Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(content): add the brief eval: 30 cases, the 2026 brief rules as a rubric, a gate on what the brief wasn't given, and a judge's quoted scores (CE-EVAL-001, brief rules PR 2) (#1538)
Cihand15419bef58f
diff --git a/server/.env.example b/server/.env.exampleindex ed9b73c1a..8d0c6b4ef 100644--- a/server/.env.example+++ b/server/.env.example@@ -55,6 +55,10 @@ COMPETITOR_SUGGESTION_MODEL=google/gemini-3-flash-preview# (improveMonthlyBudgetUsd). Falls back to# DEFAULT_SUGGESTION_MODEL, then google/gemini-3-flash-preview.# IMPROVE_SUGGESTION_MODEL=google/gemini-3-flash-preview+# The brief eval's judge (node src/scripts/eval-brief.js, maintainers only):+# another model family than the brief's, so it doesn't grade its own. Falls+# back to anthropic/claude-haiku-4-5 (needs ANTHROPIC_API_KEY).+# BRIEF_EVAL_JUDGE_MODEL=# Content Editor Quality's AI checks (CE-046 on: the kind of a source's# site the rules can't place): a small, cheap model. Metered from the same# AI budget as "quality". Falls back to DEFAULT_SUGGESTION_MODEL, thendiff --git a/server/.gitignore b/server/.gitignoreindex c9ea12fb3..cec113ec9 100644--- a/server/.gitignore+++ b/server/.gitignore@@ -8,3 +8,6 @@ yarn-error.log*.DS_Storecoverage/.nyc_output/++# The brief eval's partial runs (src/scripts/eval-brief.js): only a whole run is the record.+src/lib/__fixtures__/brief-eval/recorded.partial.jsondiff --git a/server/src/lib/__fixtures__/brief-eval/cases.js b/server/src/lib/__fixtures__/brief-eval/cases.jsnew file mode 100644index 000000000..81b376860--- /dev/null+++ b/server/src/lib/__fixtures__/brief-eval/cases.js@@ -0,0 +1,449 @@+/**+ * The brief eval's cases (CE-EVAL-001): 30 pages to plan, ten per site type+ * (B2B, B2C, e-commerce), in English, Turkish and German, over the page+ * types each site type has, new pages and live ones. Each is what Rebuild+ * outline is given: the brand, the draft, the prompts and the Sources tab's+ * fan-outs, Search Console queries, cited pages, the live page's headings+ * and the length its rules ask for. Made-up brands and pages only: no+ * customer's.+ *+ * `node src/scripts/eval-brief.js` writes each one's brief and the judge's+ * scores beside them (recorded.json).+ */++const BRANDS = {+ b2b: {+ en: {+ name: 'Ledgerly',+ industry: 'Finance software',+ description: 'Cash forecasting for small finance teams.',+ },+ tr: {+ name: 'Faturam',+ industry: 'Muhasebe yazılımı',+ description: 'KOBİ’ler için e-fatura ve tahsilat takibi.',+ },+ de: {+ name: 'Lagerwerk',+ industry: 'Lagerverwaltung',+ description: 'Lagersoftware für kleine Händler und Manufakturen.',+ },+ },+ b2c: {+ en: {+ name: 'Trailhead Tours',+ industry: 'Travel',+ description: 'Small-group hiking trips in Europe.',+ },+ tr: {+ name: 'Kapadokya Rotaları',+ industry: 'Seyahat',+ description: 'Kapadokya’da küçük gruplarla yürüyüş ve balon turları.',+ },+ de: {+ name: 'Bergzeit Reisen',+ industry: 'Reisen',+ description: 'Wanderreisen in den Alpen für kleine Gruppen.',+ },+ },+ ecommerce: {+ en: {+ name: 'Velo Studio',+ industry: 'Bicycles',+ description: 'City and gravel bikes, built to order.',+ },+ tr: {+ name: 'Atölye Seramik',+ industry: 'Ev ve yaşam',+ description: 'El yapımı seramik kupa, tabak ve vazolar.',+ },+ de: {+ name: 'Laufwerk',+ industry: 'Schuhe',+ description: 'Laufschuhe für Straße und Trail.',+ },+ },+};++const LENGTHS = { short: 3500, medium: 6000, long: 9000 };++/** A case as Rebuild outline gets it: the brand with its language and site type. */+function kase(siteType, language, pageType, page) {+ const brand = { ...BRANDS[siteType][language], language, site_type: siteType };+ return {+ id: `${siteType}-${language}-${pageType}`,+ siteType,+ language,+ brand,+ draft: {+ title: page.title,+ page_type: pageType,+ method: page.live ? 'optimize' : 'create',+ source_url: page.live ?? null,+ },+ prompts: page.prompts,+ sources: {+ promptIds: [],+ fanouts: page.fanouts.map(([query, ...platforms]) => ({ query, platforms })),+ pages: (page.cited ?? []).map(([url, kind]) => ({ url, kind })),+ queries: page.queries ?? [],+ headings: page.headings ?? [],+ length: LENGTHS[page.length ?? 'medium'],+ },+ };+}++export const CASES = [+ // B2B, English.+ kase('b2b', 'en', 'feature_page', {+ title: 'Rolling cash forecasts',+ prompts: ['How do I build a rolling 13-week cash forecast?', 'Best cash forecasting software'],+ fanouts: [+ ['13 week cash flow forecast template', 'chatgpt', 'perplexity-web'],+ ['rolling cash forecast vs static budget', 'copilot-web'],+ ['cash forecasting software small business', 'chatgpt'],+ ['how accurate are cash forecasts', 'perplexity-web'],+ ],+ queries: ['rolling cash forecast', '13 week cash flow model'],+ cited: [+ ['https://www.reddit.com/r/FPandA/comments/abc', 'forum'],+ ['https://www.example-finance-blog.com/13-week-cash-flow', 'blog'],+ ],+ }),+ kase('b2b', 'en', 'comparison', {+ title: 'Ledgerly vs spreadsheets for cash forecasting',+ prompts: ['Is cash forecasting software better than Excel?'],+ fanouts: [+ ['excel cash flow forecast limitations', 'chatgpt'],+ ['cash forecasting tool vs spreadsheet', 'perplexity-web', 'copilot-web'],+ ['when to stop using spreadsheets for finance', 'chatgpt'],+ ],+ queries: ['cash forecasting excel vs software'],+ cited: [['https://www.g2.com/categories/cash-flow-management', 'review']],+ }),+ kase('b2b', 'en', 'guide', {+ title: 'How to reconcile a cash forecast with the bank',+ live: 'https://ledgerly.example/guides/reconcile-forecast',+ headings: ['Why reconcile', 'Step 1: Export the bank feed', 'Step 2: Match payments'],+ prompts: ['How do I compare my cash forecast to actuals?'],+ fanouts: [+ ['forecast vs actual cash variance analysis', 'chatgpt', 'perplexity-web'],+ ['bank reconciliation for cash forecasting', 'copilot-web'],+ ['cash forecast variance threshold', 'chatgpt'],+ ],+ queries: ['cash forecast variance', 'forecast to actual reconciliation'],+ length: 'long',+ }),+ kase('b2b', 'en', 'pricing', {+ title: 'Ledgerly pricing',+ live: 'https://ledgerly.example/pricing',+ headings: ['Plans', 'FAQ'],+ prompts: ['How much does cash forecasting software cost?'],+ fanouts: [+ ['cash forecasting software pricing', 'chatgpt'],+ ['ledgerly pricing per user', 'perplexity-web'],+ ['free cash flow forecasting tool', 'copilot-web'],+ ],+ length: 'short',+ }),+ // B2B, Turkish.+ kase('b2b', 'tr', 'definition', {+ title: 'E-fatura nedir?',+ prompts: ['E-fatura nedir, kimler kullanmak zorunda?', 'E-fatura ile e-arşiv farkı ne?'],+ fanouts: [+ ['e-fatura zorunluluğu kimleri kapsar', 'chatgpt', 'perplexity-web'],+ ['e-fatura e-arşiv farkı', 'chatgpt'],+ ['e-fatura nasıl kesilir', 'copilot-web'],+ ['e-fatura entegratörü nedir', 'perplexity-web'],+ ],+ queries: ['e-fatura nedir', 'e-fatura zorunluluğu'],+ cited: [['https://www.ornek-muhasebe-blogu.com/e-fatura-rehberi', 'blog']],+ }),+ kase('b2b', 'tr', 'integration', {+ title: 'Faturam ve pazaryeri entegrasyonu',+ prompts: ['Pazaryeri siparişlerini otomatik faturalamak mümkün mü?'],+ fanouts: [+ ['pazaryeri e-fatura entegrasyonu', 'chatgpt'],+ ['sipariş geldiğinde otomatik fatura kesme', 'perplexity-web', 'chatgpt'],+ ['e-fatura api entegrasyonu', 'copilot-web'],+ ],+ queries: ['pazaryeri fatura entegrasyonu'],+ }),+ kase('b2b', 'tr', 'docs_faq', {+ title: 'Tahsilat takibi: sık sorulanlar',+ live: 'https://faturam.example/yardim/tahsilat',+ headings: ['Hatırlatma nasıl gönderilir?', 'Kısmi ödeme'],+ prompts: ['Geciken ödemeler için otomatik hatırlatma nasıl kurulur?'],+ fanouts: [+ ['geciken ödeme hatırlatma e-postası', 'chatgpt'],+ ['tahsilat takibi nasıl yapılır', 'perplexity-web'],+ ['kısmi ödeme nasıl kaydedilir', 'copilot-web'],+ ],+ length: 'short',+ }),+ // B2B, German.+ kase('b2b', 'de', 'solution_page', {+ title: 'Lagersoftware für Manufakturen',+ prompts: ['Welche Lagersoftware eignet sich für kleine Manufakturen?'],+ fanouts: [+ ['lagerverwaltung kleine manufaktur', 'chatgpt', 'perplexity-web'],+ ['chargenverfolgung lagersoftware', 'copilot-web'],+ ['lagersoftware mit shopify anbindung', 'chatgpt'],+ ],+ queries: ['lagersoftware manufaktur'],+ cited: [['https://www.beispiel-handel-forum.de/thread/123', 'forum']],+ }),+ kase('b2b', 'de', 'alternatives', {+ title: 'Alternativen zur Lagerverwaltung in Excel',+ prompts: ['Was ist besser als Excel für die Lagerverwaltung?'],+ fanouts: [+ ['excel lagerverwaltung grenzen', 'chatgpt'],+ ['einfache lagersoftware alternative', 'perplexity-web', 'copilot-web'],+ ['kostenlose lagerverwaltung', 'chatgpt'],+ ],+ }),+ kase('b2b', 'de', 'blog_post', {+ title: 'Inventur im Kleinbetrieb: so geht es schneller',+ live: 'https://lagerwerk.example/blog/inventur',+ headings: ['Stichtagsinventur', 'Permanente Inventur'],+ prompts: ['Wie mache ich eine Inventur im kleinen Betrieb?'],+ fanouts: [+ ['stichtagsinventur oder permanente inventur', 'chatgpt', 'perplexity-web'],+ ['inventur mit barcode scanner', 'copilot-web'],+ ['inventur checkliste', 'chatgpt'],+ ],+ queries: ['inventur kleinbetrieb', 'inventur checkliste'],+ length: 'long',+ }),+ // B2C, English.+ kase('b2c', 'en', 'destination_page', {+ title: 'Hiking in the Dolomites',+ prompts: [+ 'When is the best time to hike in the Dolomites?',+ 'Best Dolomites hikes for beginners',+ ],+ fanouts: [+ ['best time to hike dolomites', 'chatgpt', 'perplexity-web'],+ ['dolomites hut to hut beginners', 'chatgpt'],+ ['where to stay dolomites hiking', 'copilot-web'],+ ['dolomites hiking without a car', 'perplexity-web'],+ ],+ queries: ['dolomites hiking', 'dolomites best time'],+ cited: [['https://www.reddit.com/r/Dolomites/comments/xyz', 'forum']],+ }),+ kase('b2c', 'en', 'itinerary', {+ title: 'Seven days on the Tour du Mont Blanc',+ prompts: ['Can you do the Tour du Mont Blanc in a week?'],+ fanouts: [+ ['tour du mont blanc 7 day itinerary', 'chatgpt'],+ ['tmb stages for beginners', 'perplexity-web', 'copilot-web'],+ ['tmb huts booking', 'chatgpt'],+ ],+ queries: ['tmb itinerary 7 days'],+ length: 'long',+ }),+ kase('b2c', 'en', 'help_faq', {+ title: 'Before your trip: questions we get',+ live: 'https://trailhead.example/help/before-your-trip',+ headings: ['What to pack', 'Cancellations'],+ prompts: ['What fitness level do guided hiking trips need?'],+ fanouts: [+ ['guided hiking trip fitness level', 'chatgpt'],+ ['hiking tour cancellation policy', 'perplexity-web'],+ ['what to pack for hut to hut hiking', 'copilot-web', 'chatgpt'],+ ],+ length: 'short',+ }),+ kase('b2c', 'en', 'best_of', {+ title: 'The best small-group hiking trips in Europe',+ prompts: ['Which small-group hiking tours in Europe are worth it?'],+ fanouts: [+ ['best guided hiking tours europe', 'chatgpt', 'perplexity-web'],+ ['small group hiking holidays', 'copilot-web'],+ ['self guided vs guided hiking tour', 'chatgpt'],+ ],+ queries: ['small group hiking tours europe'],+ cited: [['https://www.example-travel-mag.com/best-hiking-tours', 'blog']],+ }),+ // B2C, Turkish.+ kase('b2c', 'tr', 'guide', {+ title: 'Kapadokya’da balon turu nasıl seçilir?',+ prompts: ['Kapadokya balon turu için en iyi zaman ne?', 'Balon turu güvenli mi?'],+ fanouts: [+ ['kapadokya balon turu en iyi ay', 'chatgpt', 'perplexity-web'],+ ['balon turu iptal olursa ne olur', 'chatgpt'],+ ['kapadokya balon turu fiyatları', 'copilot-web'],+ ],+ queries: ['kapadokya balon turu', 'balon turu ne zaman'],+ }),+ kase('b2c', 'tr', 'help_faq', {+ title: 'İptal ve iade: sık sorulanlar',+ live: 'https://kapadokyarotalari.example/iptal-iade',+ headings: ['İptal', 'İade'],+ prompts: ['Tur iptal edilirse ücret iade edilir mi?'],+ fanouts: [+ ['tur iptali ücret iadesi', 'chatgpt'],+ ['hava koşulları nedeniyle iptal edilen tur', 'perplexity-web'],+ ],+ length: 'short',+ }),+ kase('b2c', 'tr', 'blog_post', {+ title: 'Kapadokya’da üç günlük yürüyüş rotası',+ prompts: ['Kapadokya’da hangi vadiler yürünür?'],+ fanouts: [+ ['güvercinlik vadisi yürüyüşü', 'chatgpt'],+ ['kızılçukur vadisi gün batımı', 'perplexity-web', 'copilot-web'],+ ['ıhlara vadisi kaç saat', 'chatgpt'],+ ],+ queries: ['kapadokya yürüyüş rotaları'],+ length: 'long',+ }),+ // B2C, German.+ kase('b2c', 'de', 'destination_page', {+ title: 'Wandern im Allgäu',+ prompts: ['Wo kann man im Allgäu am schönsten wandern?'],+ fanouts: [+ ['allgäu wanderungen leicht', 'chatgpt', 'perplexity-web'],+ ['allgäu hüttenwanderung mehrtägig', 'copilot-web'],+ ['beste reisezeit allgäu wandern', 'chatgpt'],+ ],+ queries: ['wandern allgäu'],+ }),+ kase('b2c', 'de', 'comparison', {+ title: 'Geführte oder selbst geführte Wanderreise?',+ prompts: ['Lohnt sich eine geführte Wanderreise?'],+ fanouts: [+ ['geführte wanderreise vorteile', 'chatgpt'],+ ['individuelle wanderreise mit gepäcktransport', 'perplexity-web', 'copilot-web'],+ ['wanderreise kosten vergleich', 'chatgpt'],+ ],+ }),+ kase('b2c', 'de', 'help_faq', {+ title: 'Häufige Fragen zu unseren Hüttentouren',+ live: 'https://bergzeit.example/faq/huettentouren',+ headings: ['Kondition', 'Gepäck'],+ prompts: ['Wie fit muss man für eine Hüttentour sein?'],+ fanouts: [+ ['hüttentour kondition anfänger', 'chatgpt'],+ ['hüttentour packliste', 'perplexity-web'],+ ['hüttenschlafsack pflicht', 'copilot-web'],+ ],+ length: 'short',+ }),+ // E-commerce, English.+ kase('ecommerce', 'en', 'product_page', {+ title: 'Gravel One',+ live: 'https://velo.example/bikes/gravel-one',+ headings: ['Specs', 'Sizing'],+ prompts: ['What is a good gravel bike for commuting?'],+ fanouts: [+ ['gravel bike for commuting', 'chatgpt', 'perplexity-web'],+ ['gravel bike tire clearance', 'copilot-web'],+ ['steel vs aluminum gravel bike', 'chatgpt'],+ ],+ queries: ['gravel bike commuting'],+ }),+ kase('ecommerce', 'en', 'buying_guide', {+ title: 'How to choose a city bike',+ prompts: ['How do I choose a city bike?', 'Do I need gears on a city bike?'],+ fanouts: [+ ['city bike buying guide', 'chatgpt'],+ ['internal gear hub vs derailleur city bike', 'perplexity-web', 'copilot-web'],+ ['city bike frame size chart', 'chatgpt'],+ ['belt drive city bike pros and cons', 'perplexity-web'],+ ],+ queries: ['how to choose a city bike'],+ cited: [['https://www.reddit.com/r/bikecommuting/comments/def', 'forum']],+ length: 'long',+ }),+ kase('ecommerce', 'en', 'sizing_guide', {+ title: 'Bike size guide',+ prompts: ['What size bike do I need for my height?'],+ fanouts: [+ ['bike size chart by height', 'chatgpt', 'perplexity-web'],+ ['between two bike sizes', 'copilot-web'],+ ['inseam measurement for bike size', 'chatgpt'],+ ],+ length: 'short',+ }),+ kase('ecommerce', 'en', 'category_page', {+ title: 'Gravel bikes',+ live: 'https://velo.example/bikes/gravel',+ headings: ['All gravel bikes'],+ prompts: ['Best gravel bikes built to order'],+ fanouts: [+ ['custom gravel bike', 'chatgpt'],+ ['gravel bike with fenders and racks', 'perplexity-web'],+ ['gravel bike frame materials', 'copilot-web', 'chatgpt'],+ ],+ queries: ['custom gravel bike'],+ }),+ // E-commerce, Turkish.+ kase('ecommerce', 'tr', 'collection_page', {+ title: 'El yapımı kupalar',+ prompts: ['El yapımı seramik kupa nereden alınır?'],+ fanouts: [+ ['el yapımı seramik kupa', 'chatgpt', 'perplexity-web'],+ ['seramik kupa bulaşık makinesine girer mi', 'chatgpt'],+ ['hediye seramik kupa', 'copilot-web'],+ ],+ queries: ['el yapımı kupa'],+ }),+ kase('ecommerce', 'tr', 'help_faq', {+ title: 'Seramik bakımı: sık sorulanlar',+ live: 'https://atolyeseramik.example/bakim',+ headings: ['Bulaşık makinesi', 'Mikrodalga'],+ prompts: ['El yapımı seramik mikrodalgada kullanılır mı?'],+ fanouts: [+ ['seramik tabak mikrodalga', 'chatgpt'],+ ['el yapımı seramik bakımı', 'perplexity-web'],+ ['sırlı seramik gıdaya uygun mu', 'copilot-web'],+ ],+ length: 'short',+ }),+ kase('ecommerce', 'tr', 'best_of', {+ title: 'Sofra için en iyi seramik tabak setleri',+ prompts: ['Günlük kullanım için hangi seramik tabak seti alınmalı?'],+ fanouts: [+ ['seramik tabak seti önerisi', 'chatgpt', 'perplexity-web'],+ ['porselen mi seramik mi', 'chatgpt'],+ ['kırılmaya dayanıklı tabak', 'copilot-web'],+ ],+ queries: ['seramik tabak seti'],+ }),+ // E-commerce, German.+ kase('ecommerce', 'de', 'product_page', {+ title: 'Trail Pro 2',+ live: 'https://laufwerk.example/schuhe/trail-pro-2',+ headings: ['Technische Daten', 'Passform'],+ prompts: ['Welcher Trailschuh ist gut für matschige Wege?'],+ fanouts: [+ ['trailschuh matsch grip', 'chatgpt', 'perplexity-web'],+ ['trailschuh sprengung', 'copilot-web'],+ ['trailschuh wasserdicht oder nicht', 'chatgpt'],+ ],+ }),+ kase('ecommerce', 'de', 'buying_guide', {+ title: 'Den richtigen Laufschuh finden',+ prompts: ['Wie finde ich den richtigen Laufschuh?'],+ fanouts: [+ ['laufschuh kaufberatung', 'chatgpt'],+ ['pronation laufschuh', 'perplexity-web', 'copilot-web'],+ ['laufschuh größe zu klein', 'chatgpt'],+ ['wie oft laufschuhe wechseln', 'perplexity-web'],+ ],+ queries: ['laufschuh kaufberatung'],+ length: 'long',+ }),+ kase('ecommerce', 'de', 'policy', {+ title: 'Versand und Rückgabe',+ live: 'https://laufwerk.example/versand-rueckgabe',+ headings: ['Versand', 'Rückgabe'],+ prompts: ['Kann ich gelaufene Schuhe zurückgeben?'],+ fanouts: [+ ['laufschuhe zurückgeben getragen', 'chatgpt'],+ ['rückgabe laufschuhe frist', 'perplexity-web'],+ ],+ length: 'short',+ }),+];diff --git a/server/src/lib/__fixtures__/brief-eval/samples.js b/server/src/lib/__fixtures__/brief-eval/samples.jsnew file mode 100644index 000000000..dfe88483a--- /dev/null+++ b/server/src/lib/__fixtures__/brief-eval/samples.js@@ -0,0 +1,111 @@+/**+ * Two briefs for the first case (b2b-en-feature_page), as the model might+ * answer: one that follows the brief rules, one that breaks them. The eval's+ * own tests read them through shapeBrief, its checks and readJudged, and+ * judge answers for them stand in for a judge model's.+ */++/** Plans each H2's answer, its facts with their kind of source, a table, an FAQ, links. */+export const GOOD = {+ goal: 'Show finance leads how a rolling 13-week cash forecast works in Ledgerly, and why it beats a static budget.',+ audience: 'Finance leads at small companies who forecast cash in spreadsheets today.',+ outline: [+ {+ heading: 'What a rolling 13-week cash forecast is',+ keyPoints: [+ 'Answer first: a 13-week forecast that moves forward one week every week, from known payments.',+ 'TL;DR at the top: three key takeaways for the reader.',+ ],+ fanout: '13 week cash flow forecast template',+ platform: 'chatgpt',+ },+ {+ heading: 'Rolling cash forecast vs a static budget, side by side',+ keyPoints: [+ 'A table comparing update cadence, accuracy and effort.',+ 'Fact to state: how far forecasts drift, from [figure, source] (a study).',+ ],+ fanout: 'rolling cash forecast vs static budget',+ platform: 'copilot-web',+ },+ {+ heading: 'How accurate cash forecasts are, and how to check yours',+ keyPoints: [+ 'Proof to gather: Ledgerly’s own variance data from customers, dated.',+ 'Add an "Updated" date and the year the figures are from.',+ ],+ fanout: 'how accurate are cash forecasts',+ platform: 'perplexity-web',+ },+ {+ heading: 'Choosing cash forecasting software for a small business',+ keyPoints: [+ 'Steps to set it up in Ledgerly, as a numbered list.',+ 'Link internally to the Ledgerly pricing page; cite the sources given.',+ ],+ fanout: 'cash forecasting software small business',+ platform: 'chatgpt',+ },+ {+ heading: 'Questions finance teams ask',+ keyPoints: ['An FAQ of the prompts, each answered in two sentences.'],+ fanout: null,+ platform: null,+ },+ ],+ keywords: ['rolling cash forecast', '13 week cash flow model', 'cash forecasting software'],+};++/**+ * Makes up a figure, a person, a quote and an address; answers one fan-out+ * of four; one section has no point.+ */+export const BAD = {+ goal: 'Rank for cash forecasting, cash forecasting software, best cash forecasting tool.',+ audience: 'Everyone.',+ outline: [+ {+ heading: 'Cash forecasting cash forecasting software best cash forecasting',+ keyPoints: [+ '73% of CFOs say forecasts fail, per https://made-up.example/report.',+ 'Dr. Jane Doe recommends Ledgerly.',+ ],+ fanout: '13 week cash flow forecast template',+ platform: 'chatgpt',+ },+ { heading: 'More', keyPoints: [], fanout: null, platform: null },+ ],+ keywords: ['cash forecasting', 'cash forecasting software', 'best cash forecasting tool'],+};++/** A judge's answer for GOOD: its words quoted, one quote it doesn't have. */+export const GOOD_JUDGED = {+ scores: [+ {+ key: 'answerFirst',+ score: 2,+ quote: 'Answer first: a 13-week forecast that moves forward one week every week',+ why: 'The first H2 says its answer.',+ },+ {+ key: 'sourcedFacts',+ score: 1,+ quote: 'Fact to state: how far forecasts drift, from [figure, source] (a study).',+ why: 'One fact, with its kind of source.',+ },+ {+ key: 'format',+ score: 2,+ quote: 'A table comparing update cadence, accuracy and effort.',+ why: 'A table and an FAQ.',+ },+ // Not the brief's words: it can't earn the score.+ {+ key: 'links',+ score: 2,+ quote: 'Link to the G2 category and the Ledgerly blog.',+ why: 'Links planned.',+ },+ { key: 'unknown', score: 2, quote: '', why: 'Not a criterion.' },+ ],+};diff --git a/server/src/lib/content-brief-eval.js b/server/src/lib/content-brief-eval.jsnew file mode 100644index 000000000..1319f83ec--- /dev/null+++ b/server/src/lib/content-brief-eval.js@@ -0,0 +1,304 @@+import { createHash } from 'node:crypto';+import { z } from 'zod';+import { HEADING_LIMITS } from './content-agent.js';+import { SYSTEM_PROMPT, briefGuard, buildBriefPrompt, outlineSchema } from './content-brief.js';+import { NAME_PLACEHOLDER, QUOTE_PLACEHOLDER } from './content-people.js';+import { qualityPhase } from './content-quality.js';+import { NUMBER_PLACEHOLDER } from './content-sources.js';++/**+ * The brief eval (CE-EVAL-001): how good the Content Editor's briefs are,+ * read against the brief rules for AI search (Google's May 2026 guide,+ * Bing's February 2026 guidelines, the GEO paper). Our own briefs only:+ * no competitor's.+ *+ * Each brief is read two ways:+ * * checks anyone can repeat (briefChecks): its shape, the fan-outs it+ * answers, and, as a gate, no figure, name, quote or address it wasn't+ * given left in it after the guard;+ * * a judge model's scores on the nine criteria (BRIEF_CRITERIA), 0 to 2,+ * each with the brief's own words that earn it (readJudged drops a+ * score whose words aren't in the brief).+ *+ * `node src/scripts/eval-brief.js` records both for the cases in+ * __fixtures__/brief-eval/cases.js (recorded.json), with a provider key: the+ * maintainer's run, never CI's. content-brief-eval.test.js reads the record+ * for free on every run, and skips it once the brief's prompt has changed+ * (promptVersion) until it is recorded again.+ */++/** The judge: another model family than the brief's, so it doesn't grade its own. */+export function briefEvalJudgeModelId() {+ return process.env.BRIEF_EVAL_JUDGE_MODEL || 'anthropic/claude-haiku-4-5';+}++/** The nine criteria, as the brief rules (2026) name them: 0 missing, 1 partly, 2 fully. */+export const BRIEF_CRITERIA = [+ {+ key: 'answerFirst',+ ask: 'Does each H2 say what its direct answer must state, so the section can open with a 40–75-word answer?',+ },+ {+ key: 'uniqueAngle',+ ask: "Does it name a unique angle or first-hand proof to gather (the brand's own data, cases, tests, experience), rather than commodity content?",+ },+ {+ key: 'sourcedFacts',+ ask: 'Does it name the facts to state, each with the kind of source it needs (a statistic, a study, a quote), about one per section, without a figure of its own?',+ },+ {+ key: 'entities',+ ask: 'Does it name the brand, its products and the competitors given consistently, exactly as given?',+ },+ {+ key: 'topEssentials',+ ask: 'Does it put the essentials at the top (a TL;DR or key takeaways) and keep the page to one topic?',+ },+ {+ key: 'freshness',+ ask: 'Does it plan freshness where the topic needs it: a dated data point, an updated date, the year?',+ },+ {+ key: 'format',+ ask: 'Does it say how each part is best shown (a table for a comparison, steps for a how-to, a question FAQ at the end), with short headings (H1 up to 60 characters, H2 about 8–15 words)?',+ },+ {+ key: 'naturalLanguage',+ ask: 'Is it free of keyword stuffing and of wording engineered to trigger citations?',+ },+ {+ key: 'links',+ ask: "Does it plan internal links and outbound sources, to the brand's own pages or the sources given only?",+ },+];++/** Sections a brief should plan, as its prompt asks: 3 to 10. */+const SECTIONS = { min: 3, max: 10 };+/** Keywords it should list, as its prompt asks: up to 10. */+const MAX_KEYWORDS = 10;++// An address: with its scheme, or a host starting "www.".+const URL_RE = /(?:https?:\/\/|\bwww\.)[^\s"'<>)\]]+/gi;++/** An address as compared: its host without "www." and in small letters, its path without the last "/". */+function urlKey(url) {+ try {+ const read = new URL(/^www\./i.test(url) ? `https://${url}` : url);+ return `${read.hostname.toLowerCase().replace(/^www\./, '')}${read.pathname.replace(/\/+$/, '')}`;+ } catch {+ return url.toLowerCase();+ }+}++/**+ * A brief's fields as the judge sees them and as a quote is read in: the+ * goal, the audience, each heading and key point, the keywords as one line;+ * each with whether it's a heading.+ */+function briefFields(brief) {+ return [+ { label: 'Goal: ', text: brief.goal, heading: false },+ { label: 'Audience: ', text: brief.audience, heading: false },+ ...brief.outline.flatMap((s) => [+ { label: 'H2: ', text: s.heading, heading: true },+ ...s.keyPoints.map((point) => ({ label: '- ', text: point, heading: false })),+ ]),+ { label: 'Keywords: ', text: brief.keywords.join(', '), heading: false },+ ].filter((f) => f.text);+}++/** A brief's words, every field, as one text. */+export function briefText(brief) {+ return briefFields(brief)+ .map((f) => f.text)+ .join('\n');+}++const PLACEHOLDERS = [NUMBER_PLACEHOLDER, NAME_PLACEHOLDER, QUOTE_PLACEHOLDER];++/**+ * The checks anyone can repeat, for one case's brief (after shapeBrief):+ * * sections, those without a key point, headings past the editor's limit,+ * and keywords past the ten the prompt asks for;+ * * fanoutCoverage: the share of the case's fan-outs an H2 answers;+ * * placeholders: the figures, names and quotes the guard took out;+ * * invented (the gate, all 0): figures and names or quotes the guard let+ * through that the case didn't give, and addresses it didn't list.+ */+export function briefChecks(brief, c) {+ const text = briefText(brief);+ const guard = briefGuard(c);+ // Each field read as the guard reads it: a heading as one, a question in quotes kept.+ const found = briefFields(brief).map((f) => guard.found(f.text, f.heading));+ const given = new Set(+ [...c.sources.pages.map((p) => p.url), c.draft.source_url].filter(Boolean).map(urlKey),+ );+ const answered = new Set(brief.outline.map((s) => s.fanout).filter(Boolean));+ return {+ sections: brief.outline.length,+ sectionsInRange: brief.outline.length >= SECTIONS.min && brief.outline.length <= SECTIONS.max,+ emptySections: brief.outline.filter((s) => s.keyPoints.length === 0).length,+ longHeadings: brief.outline.filter((s) => s.heading.length > HEADING_LIMITS.other).length,+ extraKeywords: Math.max(0, brief.keywords.length - MAX_KEYWORDS),+ fanoutCoverage: c.sources.fanouts.length+ ? c.sources.fanouts.filter((f) => answered.has(f.query)).length / c.sources.fanouts.length+ : null,+ placeholders: PLACEHOLDERS.reduce((n, p) => n + text.split(p).length - 1, 0),+ invented: {+ figures: found.flatMap((f) => f.figures),+ people: found.flatMap((f) => f.people),+ urls: [...text.matchAll(URL_RE)]+ .map((m) => m[0].replace(/[.,;:!?]+$/, ''))+ .filter((url) => !given.has(urlKey(url))),+ },+ };+}++/** Whether a brief passes the gate: nothing it wasn't given left in it. */+export function passesGate(checks) {+ const { figures, people, urls } = checks.invented;+ return figures.length === 0 && people.length === 0 && urls.length === 0;+}++/** What the judge answers: per criterion, its score, the brief's words for it, and why. */+export const judgeSchema = z.object({+ scores: z.array(+ z.object({+ key: z.enum(BRIEF_CRITERIA.map((k) => k.key)),+ score: z.number(),+ quote: z.string(),+ why: z.string(),+ }),+ ),+});++export const JUDGE_SYSTEM_PROMPT = `You grade one content brief: the plan a writer follows to write a web page that AI assistants (ChatGPT, Perplexity, Gemini, Copilot, Google's AI Overviews) cite.++Rules:+- Score each criterion listed 0 (missing), 1 (partly) or 2 (fully), from what the brief says only. Don't reward length or fill in what it leaves out.+- For a score above 0, put in "quote" the brief's own words that earn it, copied exactly from one of its lines, at least 3 words and at most 30. For 0, leave "quote" empty.+- A placeholder in square brackets, e.g. [figure, source], marks a fact for the writer to find: that is correct, not a missing fact.+- In "why", one short sentence.+- Score every criterion listed, once each, by its key.`;++/** What the judge is told: the case the brief was built for, the brief, the criteria. */+export function buildJudgePrompt(c, brief) {+ const list = (items) => (items.length ? items.map((item) => `- ${item}`).join('\n') : '- none');+ return `The page: ${c.draft.title} (${c.draft.page_type}, ${c.draft.method === 'optimize' ? 'an existing page' : 'a new page'}), for ${c.brand.name}: ${c.brand.description}+Written in: ${c.language}++Prompts it must answer:+${list(c.prompts)}++AI searches (fan-outs) it was given:+${list(c.sources.fanouts.map((f) => f.query))}++The brief:+${briefFields(brief)+ .map((f) => `${f.label}${f.text}`)+ .join('\n')}++Criteria:+${BRIEF_CRITERIA.map((k) => `- ${k.key}: ${k.ask}`).join('\n')}`;+}++/** Text as a quote is compared: one form of each character, straight quotes, small letters, single spaces. */+const norm = (text) =>+ String(text ?? '')+ .normalize('NFKC')+ .replace(/[‘’]/g, "'")+ .replace(/[“”„«»]/g, '"')+ .toLowerCase()+ .replace(/\s+/g, ' ')+ .trim();+const EDGES = /^["'\s.,;:!?-]+|["'\s.,;:!?-]+$/g;+/** A quote's parts: without the marks around it, split where it was shortened ("…"). */+const quoteParts = (quote) =>+ norm(quote)+ .split(/\s*(?:…|\.\.\.)\s*/)+ .map((part) => part.replace(EDGES, ''))+ .filter(Boolean);++/**+ * The judge's answer as kept: one score per criterion, a whole number 0 to+ * 2 (a criterion it left out is 0); a score above 0 whose quote isn't the+ * brief's own words is 0, marked `unquoted`, so a judge can't vouch for what+ * the brief doesn't say.+ */+export function readJudged(answer, brief) {+ const fields = briefFields(brief).map((f) => norm(f.text));+ const byKey = new Map((answer?.scores ?? []).map((s) => [s.key, s]));+ return Object.fromEntries(+ BRIEF_CRITERIA.map(({ key }) => {+ const s = byKey.get(key);+ const score = Math.max(0, Math.min(2, Math.round(Number(s?.score) || 0)));+ const parts = quoteParts(s?.quote);+ // Three words at least, each part within one line of the brief.+ const quoted =+ parts.join(' ').split(' ').length >= 3 &&+ parts.every((part) => fields.some((field) => field.includes(part)));+ const unquoted = score > 0 && !quoted;+ return [+ key,+ {+ score: unquoted ? 0 : score,+ quote: s?.quote ?? '',+ why: s?.why ?? '',+ ...(unquoted ? { unquoted: true } : {}),+ },+ ];+ }),+ );+}++/**+ * The brief's prompt as recorded: a hash of its rules, its schema and the+ * prompt each case gets, under Quality's phase (the site type line shows+ * from phase 2). A record made under another one is stale.+ */+export function promptVersion(cases, phase = qualityPhase()) {+ const before = process.env.CONTENT_QUALITY_PHASE;+ process.env.CONTENT_QUALITY_PHASE = String(phase);+ try {+ const hash = createHash('sha256')+ .update(SYSTEM_PROMPT)+ .update(JSON.stringify(z.toJSONSchema(outlineSchema)));+ for (const c of cases) hash.update(buildBriefPrompt(c));+ return hash.digest('hex').slice(0, 12);+ } finally {+ if (before === undefined) delete process.env.CONTENT_QUALITY_PHASE;+ else process.env.CONTENT_QUALITY_PHASE = before;+ }+}++/** A record's sums: per criterion its mean score, the gate, the checks, the tokens. */+export function summarize(cases) {+ const results = Object.values(cases);+ const mean = (values) =>+ values.length ? Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 100) / 100 : 0;+ const coverage = results.map((r) => r.checks.fanoutCoverage).filter((v) => v !== null);+ return {+ cases: results.length,+ criteria: Object.fromEntries(+ BRIEF_CRITERIA.map(({ key }) => [key, mean(results.map((r) => r.judged[key].score))]),+ ),+ total: mean(+ results.map((r) => BRIEF_CRITERIA.reduce((n, { key }) => n + r.judged[key].score, 0)),+ ),+ gateFailures: results.filter((r) => !passesGate(r.checks)).length,+ sectionsInRange: results.filter((r) => r.checks.sectionsInRange).length,+ fanoutCoverage: mean(coverage),+ placeholders: results.reduce((n, r) => n + r.checks.placeholders, 0),+ unquoted: results.reduce(+ (n, r) => n + Object.values(r.judged).filter((j) => j.unquoted).length,+ 0,+ ),+ tokens: results.reduce(+ (n, r) => ({+ input: n.input + (r.usage?.inputTokens ?? 0),+ output: n.output + (r.usage?.outputTokens ?? 0),+ }),+ { input: 0, output: 0 },+ ),+ };+}diff --git a/server/src/lib/content-brief-eval.test.js b/server/src/lib/content-brief-eval.test.jsnew file mode 100644index 000000000..37a76b94f--- /dev/null+++ b/server/src/lib/content-brief-eval.test.js@@ -0,0 +1,278 @@+import { existsSync, readFileSync } from 'fs';+import { describe, expect, it, vi } from 'vitest';+import { CASES } from './__fixtures__/brief-eval/cases.js';+import { BAD, GOOD, GOOD_JUDGED } from './__fixtures__/brief-eval/samples.js';++/**+ * The brief eval (CE-EVAL-001): its cases, its checks, how a judge's answer+ * is kept, and, once recorded (node src/scripts/eval-brief.js, with a+ * provider key: never in CI), the models' own briefs read for free.+ */++vi.mock('../config/supabase.js', () => ({ default: {} }));++const { PAGE_TYPE_FORMATS, SITE_PAGE_TYPES } = await import('./content-page-types.js');+const { shapeBrief } = await import('./content-brief.js');+const {+ BRIEF_CRITERIA,+ briefChecks,+ buildJudgePrompt,+ passesGate,+ promptVersion,+ readJudged,+ summarize,+} = await import('./content-brief-eval.js');+const { NAME_PLACEHOLDER } = await import('./content-people.js');+const { NUMBER_PLACEHOLDER } = await import('./content-sources.js');++const [FIRST] = CASES;++describe('the brief eval’s cases', () => {+ it('holds 30 pages: ten per site type, in English, Turkish and German, new and live', () => {+ expect(CASES).toHaveLength(30);+ for (const site of ['b2b', 'b2c', 'ecommerce']) {+ expect(CASES.filter((c) => c.siteType === site)).toHaveLength(10);+ }+ expect(new Set(CASES.map((c) => c.language))).toEqual(new Set(['en', 'tr', 'de']));+ expect(new Set(CASES.map((c) => c.id)).size).toBe(CASES.length);+ expect(new Set(CASES.map((c) => c.draft.method))).toEqual(new Set(['create', 'optimize']));+ });++ it('gives each what Rebuild outline is given: a page type, prompts and fan-outs', () => {+ for (const c of CASES) {+ expect(PAGE_TYPE_FORMATS[c.draft.page_type], c.id).toBeTruthy();+ // One the editor offers its site type.+ expect(SITE_PAGE_TYPES[c.siteType], c.id).toContain(c.draft.page_type);+ expect(c.prompts.length, c.id).toBeGreaterThan(0);+ expect(c.sources.fanouts.length, c.id).toBeGreaterThanOrEqual(2);+ expect(c.brand.language).toBe(c.language);+ // A live page says where it is and what its headings are.+ if (c.draft.method === 'optimize') expect(c.sources.headings.length, c.id).toBeGreaterThan(0);+ }+ });+});++describe('the rubric', () => {+ it('reads the nine brief rules, each once', () => {+ expect(BRIEF_CRITERIA.map((k) => k.key)).toEqual([+ 'answerFirst',+ 'uniqueAngle',+ 'sourcedFacts',+ 'entities',+ 'topEssentials',+ 'freshness',+ 'format',+ 'naturalLanguage',+ 'links',+ ]);+ });++ it('tells the judge the case, the brief and each criterion', () => {+ const brief = shapeBrief(GOOD, FIRST);+ const prompt = buildJudgePrompt(FIRST, brief);+ expect(prompt).toContain('Rolling cash forecasts (feature_page, a new page), for Ledgerly');+ expect(prompt).toContain('- rolling cash forecast vs static budget');+ // The brief as plain lines: what a quote is read in.+ expect(prompt).toContain('H2: What a rolling 13-week cash forecast is');+ expect(prompt).toContain('Keywords: rolling cash forecast, 13 week cash flow model');+ for (const { key } of BRIEF_CRITERIA) expect(prompt).toContain(`- ${key}: `);+ });+});++describe('the checks anyone can repeat', () => {+ it('pass a brief that answers every fan-out with nothing it wasn’t given', () => {+ const checks = briefChecks(shapeBrief(GOOD, FIRST), FIRST);+ expect(checks).toMatchObject({+ sections: 5,+ sectionsInRange: true,+ emptySections: 0,+ longHeadings: 0,+ extraKeywords: 0,+ fanoutCoverage: 1,+ // Its own placeholder for a fact to find.+ placeholders: 1,+ invented: { figures: [], people: [], urls: [] },+ });+ expect(passesGate(checks)).toBe(true);+ });++ it('find what the guard put as placeholders, an address no one gave, and a thin outline', () => {+ const brief = shapeBrief(BAD, FIRST);+ expect(brief.outline[0].keyPoints).toEqual([+ `${NUMBER_PLACEHOLDER} of CFOs say forecasts fail, per https://made-up.example/report.`,+ `${NAME_PLACEHOLDER} recommends Ledgerly.`,+ ]);+ const checks = briefChecks(brief, FIRST);+ expect(checks).toMatchObject({+ sections: 2,+ sectionsInRange: false,+ emptySections: 1,+ fanoutCoverage: 0.25,+ placeholders: 2,+ invented: { figures: [], people: [], urls: ['https://made-up.example/report'] },+ });+ expect(passesGate(checks)).toBe(false);+ });++ it('fail the gate on what a guard let through: a figure or a person it wasn’t given', () => {+ // The brief as the model wrote it, before the guard.+ const checks = briefChecks(BAD, FIRST);+ expect(checks.invented.figures).toContain('73');+ expect(checks.invented.people.join(' ')).toContain('Jane Doe');+ expect(passesGate(checks)).toBe(false);+ });++ it('read each line as the guard does: a heading as one, a question in quotes kept', () => {+ const brief = shapeBrief(+ {+ ...GOOD,+ outline: [+ {+ heading: 'What Every CFO Should Know About Cash Forecasts',+ keyPoints: ['Answer "Can Ledgerly replace Excel for a cash forecast?" first.'],+ fanout: null,+ platform: null,+ },+ ...GOOD.outline.slice(1),+ ],+ },+ FIRST,+ );+ expect(briefChecks(brief, FIRST).invented).toEqual({ figures: [], people: [], urls: [] });+ });++ it('read an address as its host and path: a given one however written, one without a scheme too', () => {+ const brief = shapeBrief(+ {+ ...GOOD,+ audience: 'Read https://WWW.Reddit.com/r/FPandA/comments/abc/ and www.made-up.example/x.',+ },+ FIRST,+ );+ expect(briefChecks(brief, FIRST).invented.urls).toEqual(['www.made-up.example/x']);+ });++ it('take an address the case gave: a cited page, or the live page', () => {+ const live = CASES.find((c) => c.draft.method === 'optimize');+ const brief = shapeBrief(+ {+ ...GOOD,+ goal: `Improve ${live.draft.source_url} for AI answers.`,+ },+ live,+ );+ expect(briefChecks(brief, live).invented.urls).toEqual([]);+ });+});++describe('a judge’s answer as kept', () => {+ it('keeps a score only with the brief’s own words, every criterion once', () => {+ const judged = readJudged(GOOD_JUDGED, shapeBrief(GOOD, FIRST));+ expect(Object.keys(judged)).toEqual(BRIEF_CRITERIA.map((k) => k.key));+ expect(judged.answerFirst).toMatchObject({ score: 2 });+ expect(judged.sourcedFacts).toMatchObject({ score: 1 });+ expect(judged.format).toMatchObject({ score: 2 });+ // Words the brief doesn't have earn nothing.+ expect(judged.links).toMatchObject({ score: 0, unquoted: true });+ // Left out: 0.+ expect(judged.freshness).toMatchObject({ score: 0, quote: '' });+ });++ it('reads a quote as written in any quotes or apostrophes, shortened, but three words at least', () => {+ const brief = shapeBrief(GOOD, FIRST);+ const read = (quote) =>+ readJudged({ scores: [{ key: 'uniqueAngle', score: 2, quote, why: '' }] }, brief).uniqueAngle;+ expect(read("Proof to gather: Ledgerly's own variance data").score).toBe(2);+ expect(read('“Proof to gather: Ledgerly’s own variance data”').score).toBe(2);+ expect(read('Proof to gather … variance data from customers').score).toBe(2);+ expect(read('Ledgerly')).toMatchObject({ score: 0, unquoted: true });+ // Across two lines: not the brief's words.+ expect(read('three key takeaways for the reader. A table comparing')).toMatchObject({+ score: 0,+ unquoted: true,+ });+ });++ it('reads a score as a whole number from 0 to 2', () => {+ const brief = shapeBrief(GOOD, FIRST);+ const quote = 'TL;DR at the top: three key takeaways for the reader.';+ const read = (score) =>+ readJudged({ scores: [{ key: 'topEssentials', score, quote, why: '' }] }, brief).topEssentials+ .score;+ expect(read(7)).toBe(2);+ expect(read(-1)).toBe(0);+ expect(read(1.4)).toBe(1);+ expect(read('2')).toBe(2);+ });++ it('sums a record: means per criterion, the gate, the checks and the tokens', () => {+ const brief = shapeBrief(GOOD, FIRST);+ const good = {+ checks: briefChecks(brief, FIRST),+ judged: readJudged(GOOD_JUDGED, brief),+ usage: { inputTokens: 2000, outputTokens: 1000 },+ };+ const badBrief = shapeBrief(BAD, FIRST);+ const bad = {+ checks: briefChecks(badBrief, FIRST),+ judged: readJudged({ scores: [] }, badBrief),+ usage: { inputTokens: 1500, outputTokens: 500 },+ };+ expect(summarize({ good, bad })).toMatchObject({+ cases: 2,+ criteria: { answerFirst: 1, sourcedFacts: 0.5, format: 1, links: 0 },+ total: 2.5,+ gateFailures: 1,+ sectionsInRange: 1,+ fanoutCoverage: 0.63,+ placeholders: 3,+ unquoted: 1,+ tokens: { input: 3500, output: 1500 },+ });+ });++ it('names the brief’s prompt by a short hash of every case’s, under Quality’s phase', () => {+ expect(promptVersion(CASES)).toMatch(/^[0-9a-f]{12}$/);+ expect(promptVersion(CASES)).toBe(promptVersion(CASES));+ expect(promptVersion(CASES.slice(1))).not.toBe(promptVersion(CASES));+ // From phase 2 the prompt names the site type.+ expect(promptVersion(CASES, 2)).not.toBe(promptVersion(CASES, 0));+ // The phase is read for it, and left as it was.+ const before = process.env.CONTENT_QUALITY_PHASE;+ promptVersion(CASES, 3);+ expect(process.env.CONTENT_QUALITY_PHASE).toBe(before);+ });+});++// The models' own briefs, once recorded: read only while the brief's prompt+// (under the Quality phase they were recorded with) is the one they were+// written under. Their checks are made again here, for free.+const recordedPath = new URL('./__fixtures__/brief-eval/recorded.json', import.meta.url);+const recorded = existsSync(recordedPath) ? JSON.parse(readFileSync(recordedPath, 'utf8')) : null;+const current =+ recorded && recorded.promptVersion === promptVersion(CASES, recorded.qualityPhase ?? 0);++describe('the recorded briefs', () => {+ it.skipIf(!current)(+ 'answer every case, leave nothing in any brief it wasn’t given, and plan 3 to 10 sections',+ () => {+ const results = CASES.map((c) => [c, recorded.cases[c.id]]);+ expect(results.filter(([, r]) => !r || r.error).map(([c]) => c.id)).toEqual([]);+ const checked = results.map(([c, r]) => [c.id, briefChecks(shapeBrief(r.object, c), c), r]);+ expect(checked.filter(([, checks]) => !passesGate(checks)).map(([id]) => id)).toEqual([]);+ const inRange = checked.filter(([, checks]) => checks.sectionsInRange).length;+ expect(inRange / checked.length).toBeGreaterThanOrEqual(0.9);+ for (const [id, , r] of checked) {+ expect(Object.keys(r.judged), id).toEqual(BRIEF_CRITERIA.map((k) => k.key));+ }+ },+ );++ it.skipIf(!recorded || current)(+ 'waits for a new record: the brief’s prompt changed since',+ () => {+ // A stale record is said, not failed: the maintainer records again.+ expect(recorded.promptVersion).not.toBe(promptVersion(CASES, recorded.qualityPhase ?? 0));+ },+ );+});diff --git a/server/src/lib/content-brief.js b/server/src/lib/content-brief.jsindex dd08108de..ecb9f03ba 100644--- a/server/src/lib/content-brief.js+++ b/server/src/lib/content-brief.js@@ -14,8 +14,8 @@ import {stopIfStopped,} from './content-runs.js';import { citedPagesLength } from './content-cited-length.js';-import { guardPeople, knownPeople } from './content-people.js';-import { knownNumbers, placeholderNumbers, todayText } from './content-sources.js';+import { guardPeople, knownPeople, peopleEdits } from './content-people.js';+import { addedNumbers, knownNumbers, placeholderNumbers, todayText } from './content-sources.js';import { reserveAiBudget } from './plan-guard.js';/**@@ -98,7 +98,7 @@ export const BRIEF_LIMITS = {// No counts or lengths enforced on the model: a miss would fail the whole// call (charged all the same). They are asked for here and cut afterwards.-const outlineSchema = z.object({+export const outlineSchema = z.object({goal: z.string().describe('What the page must achieve for the brand, in one sentence.'),audience: z.string().describe('Who the page is for, in one sentence.'),outline: z@@ -120,7 +120,7 @@ const outlineSchema = z.object({keywords: z.array(z.string()).describe('Up to 10 keywords.'),});-const SYSTEM_PROMPT = `You are a senior content strategist for Answer Engine Optimization (AEO). You plan one web page so AI assistants (ChatGPT, Perplexity, Gemini, Copilot, Google's AI Overviews) cite it.+export const SYSTEM_PROMPT = `You are a senior content strategist for Answer Engine Optimization (AEO). You plan one web page so AI assistants (ChatGPT, Perplexity, Gemini, Copilot, Google's AI Overviews) cite it.Rules:- Plan an outline of H2 sections. Each H2 should answer one of the AI searches (fan-outs) listed, the most searched first, and name it exactly in "fanout" with one of its platform keys in "platform". An H2 that answers none has null for both.@@ -257,9 +257,38 @@ async function briefWithRun({ draft, draftId, brand, prompts, sources, run }) {// Each H2 matched to the fan-out it answers (mapOutline).await advanceRun(run, 2);- // What it adds beyond what it was built from goes as a placeholder, before the cut.- const guard = briefGuard({ brand, draft, prompts, sources });const brief = {+ ...shapeBrief(object, { brand, draft, prompts, sources }),+ origin: 'rebuild',+ builtAt: new Date().toISOString(),+ builtFrom: {+ prompts: prompts.length,+ fanouts: sources.fanouts.length,+ pages: sources.pages.length,+ queries: sources.queries.length,+ },+ citedLength: await citedLength,+ };+ // Stopped meanwhile: the brief stays as it was.+ await stopIfStopped(run);+ const { error: saveError } = await supabaseAdmin+ .from('content_drafts')+ .update({ brief })+ .eq('id', draftId);+ if (saveError) throw saveError;+ return brief;+}++/**+ * The brief as saved, from the model's answer: what it adds beyond what it+ * was built from as a placeholder (briefGuard), before the cut; each H2+ * matched to the fan-out it answers (mapOutline); the keywords kept once.+ * The brief eval (content-brief-eval.js) reads the model's answers through+ * it too.+ */+export function shapeBrief(object, { brand, draft, prompts, sources }) {+ const guard = briefGuard({ brand, draft, prompts, sources });+ return {goal: clip(guard.text(object.goal), BRIEF_LIMITS.goal),audience: clip(guard.text(object.audience), BRIEF_LIMITS.audience),outline: mapOutline(@@ -278,24 +307,7 @@ async function briefWithRun({ draft, draftId, brand, prompts, sources, run }) {.filter(Boolean),),].slice(0, BRIEF_LIMITS.keywords),- origin: 'rebuild',- builtAt: new Date().toISOString(),- builtFrom: {- prompts: prompts.length,- fanouts: sources.fanouts.length,- pages: sources.pages.length,- queries: sources.queries.length,- },- citedLength: await citedLength,};- // Stopped meanwhile: the brief stays as it was.- await stopIfStopped(run);- const { error: saveError } = await supabaseAdmin- .from('content_drafts')- .update({ brief })- .eq('id', draftId);- if (saveError) throw saveError;- return brief;}/**@@ -306,9 +318,42 @@ async function briefWithRun({ draft, draftId, brand, prompts, sources, run }) {* figure or a person made up in one would pass the writer's own guard.* `text` and `heading` put each one added as a placeholder, as a written* page's are, a question in quotes aside; `keeps` drops a keyword that adds- * one, as a placeholder can't be searched for.+ * one, as a placeholder can't be searched for; `found` lists what a text+ * adds, read the same way (the brief eval's gate).*/export function briefGuard({ brand, draft, prompts, sources }) {+ const { given, known, people } = briefKnown({ brand, draft, prompts, sources });+ // The names and quotes a text may have: a question in quotes in it too.+ const askedIn = (words) => {+ const questions = [...words.matchAll(QUOTED_QUESTION)].map((m) => m[1]);+ return questions.length ? knownPeople(...given, ...questions) : people;+ };+ const text = (words, heading = false) => {+ const numbered = placeholderNumbers(words ?? '', known);+ return guardPeople(numbered, askedIn(numbered), { heading }).text;+ };+ return {+ text,+ heading: (words) => text(words, true),+ keeps: (keyword) => text(keyword) === String(keyword ?? ''),+ /** What a text says that it wasn't given, as the guard reads it: what it would take out. */+ found: (words, heading = false) => {+ const read = String(words ?? '');+ return {+ figures: addedNumbers(read, known).map((n) => read.slice(n.start, n.end)),+ people: peopleEdits(read, askedIn(read), { heading }).map((e) =>+ read.slice(e.start, e.end),+ ),+ };+ },+ };+}++/**+ * What a brief is built from (`given`), and the numbers (`known`) and the+ * names and quotes (`people`) it may say: briefGuard's, and the brief eval's.+ */+export function briefKnown({ brand, draft, prompts, sources }) {const given = [brand?.name,brand?.industry,@@ -320,19 +365,7 @@ export function briefGuard({ brand, draft, prompts, sources }) {...sources.headings,String(sources.length),];- const known = knownNumbers(...given, todayText());- const people = knownPeople(...given);- const text = (words, heading = false) => {- const numbered = placeholderNumbers(words ?? '', known);- const questions = [...numbered.matchAll(QUOTED_QUESTION)].map((m) => m[1]);- const asked = questions.length ? knownPeople(...given, ...questions) : people;- return guardPeople(numbered, asked, { heading }).text;- };- return {- text,- heading: (words) => text(words, true),- keeps: (keyword) => text(keyword) === String(keyword ?? ''),- };+ return { given, known: knownNumbers(...given, todayText()), people: knownPeople(...given) };}/**diff --git a/server/src/scripts/eval-brief.js b/server/src/scripts/eval-brief.jsnew file mode 100644index 000000000..1748d1692--- /dev/null+++ b/server/src/scripts/eval-brief.js@@ -0,0 +1,130 @@+/**+ * Record the brief eval (CE-EVAL-001, content-brief-eval.js): for each case+ * in src/lib/__fixtures__/brief-eval/cases.js, the brief Rebuild outline+ * writes (the brief's own model, prompt and shaping), the checks anyone can+ * repeat, and a judge model's scores on the nine criteria, for+ * content-brief-eval.test.js to read on every test run.+ *+ * Run from server/, with .env holding the providers' keys for both models+ * (the brief's is IMPROVE_SUGGESTION_MODEL, else DEFAULT_SUGGESTION_MODEL,+ * else google/gemini-3-flash-preview; the judge's BRIEF_EVAL_JUDGE_MODEL,+ * else anthropic/claude-haiku-4-5) and the Supabase keys the modules load+ * with (nothing is read or written there):+ * node src/scripts/eval-brief.js [count]+ * count defaults to every case: two calls each, the brief and its judge.+ *+ * Writes src/lib/__fixtures__/brief-eval/recorded.json for a whole run (every+ * case answered), else recorded.partial.json beside it (not committed), and+ * prints the sums. A case that fails is recorded with its error, never left+ * out. Reads and writes no database, charges no workspace's AI usage, never+ * prints a key, and isn't run in CI:+ * the answers it keeps are the models' of the day, for a person to record+ * again when the brief's prompt or the models change (the test skips a+ * record made under another prompt or Quality phase: run it with the phase+ * the API runs with, CONTENT_QUALITY_PHASE). Under a dollar at today's+ * prices.+ */+import 'dotenv/config';+import { writeFileSync } from 'fs';+import { generateObject } from 'ai';+import { CASES } from '../lib/__fixtures__/brief-eval/cases.js';+import { resolveModel } from '../lib/ai-provider.js';+import {+ SYSTEM_PROMPT,+ buildBriefPrompt,+ contentBriefModelId,+ outlineSchema,+ shapeBrief,+} from '../lib/content-brief.js';+import { qualityPhase } from '../lib/content-quality.js';+import {+ JUDGE_SYSTEM_PROMPT,+ briefChecks,+ briefEvalJudgeModelId,+ buildJudgePrompt,+ judgeSchema,+ passesGate,+ promptVersion,+ readJudged,+ summarize,+} from '../lib/content-brief-eval.js';++const count = Math.max(+ 1,+ Math.min(CASES.length, Number.parseInt(process.argv[2] ?? `${CASES.length}`, 10) || CASES.length),+);++/** How long one call may take before the case is recorded as failed. */+const CALL_TIMEOUT_MS = 120_000;++const modelId = contentBriefModelId();+const judgeModelId = briefEvalJudgeModelId();+const model = resolveModel(modelId);+const judge = resolveModel(judgeModelId);+const recorded = {};+const answered = {};+const started = Date.now();++for (const c of CASES.slice(0, count)) {+ try {+ const written = await generateObject({+ model,+ schema: outlineSchema,+ system: SYSTEM_PROMPT,+ prompt: buildBriefPrompt(c),+ abortSignal: AbortSignal.timeout(CALL_TIMEOUT_MS),+ });+ const brief = shapeBrief(written.object, c);+ // The judge at its model's own temperature: newer models take no other.+ const graded = await generateObject({+ model: judge,+ schema: judgeSchema,+ system: JUDGE_SYSTEM_PROMPT,+ prompt: buildJudgePrompt(c, brief),+ abortSignal: AbortSignal.timeout(CALL_TIMEOUT_MS),+ });+ const judged = readJudged(graded.object, brief);+ // The model's own answer is kept: the test shapes and checks it again for free.+ recorded[c.id] = {+ object: written.object,+ usage: written.usage,+ judgeUsage: graded.usage,+ judged,+ };+ answered[c.id] = { checks: briefChecks(brief, c), judged, usage: written.usage };+ const gate = passesGate(answered[c.id].checks) ? 'passes' : 'FAILS';+ console.log(`${c.id}: ${brief.outline.length} sections, gate ${gate}`);+ } catch (err) {+ // Kept and counted: a record that leaves a case out reads better than it is.+ recorded[c.id] = { error: String(err?.message ?? err).slice(0, 300) };+ console.error(`${c.id}: failed (${recorded[c.id].error})`);+ }+}++if (!Object.keys(answered).length) {+ console.error('No case was answered: nothing is written.');+ process.exit(1);+}+const failed = Object.keys(recorded).filter((id) => recorded[id].error);+// Only a whole run, every case answered, is the record; another goes beside it.+const whole = count === CASES.length && failed.length === 0;+const file = whole ? 'recorded.json' : 'recorded.partial.json';+writeFileSync(+ new URL(`../lib/__fixtures__/brief-eval/${file}`, import.meta.url),+ `${JSON.stringify(+ {+ model: modelId,+ judgeModel: judgeModelId,+ qualityPhase: qualityPhase(),+ promptVersion: promptVersion(CASES),+ recordedAt: new Date().toISOString(),+ cases: recorded,+ },+ null,+ 2,+ )}\n`,+);+console.log(JSON.stringify(summarize(answered), null, 2));+console.log(+ `Answered ${Object.keys(answered).length} of ${count} cases (${failed.length} failed) with ${modelId}, judged by ${judgeModelId}, in ${((Date.now() - started) / 1000).toFixed(0)} s: ${file}.`,+);