Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
test(content): add the no-fabrication test set, and close the gaps it found in the people and number guards (#1456)
Cihanefa9152c5374
diff --git a/server/src/lib/__fixtures__/no-fabrication/cases.js b/server/src/lib/__fixtures__/no-fabrication/cases.jsnew file mode 100644index 000000000..79741c234--- /dev/null+++ b/server/src/lib/__fixtures__/no-fabrication/cases.js@@ -0,0 +1,814 @@+/**+ * The no-fabrication test set (Content Editor Quality spec §9, "A test set+ * of 50 drafts shows zero invented names, quotes or numbers"): 63 drafts,+ * three site types (B2B, B2C, e-commerce) in three languages (English,+ * Turkish, German), seven answers each. Each answer is the Content Agent's,+ * as recorded or written to try the guards: it names people nobody gave+ * it, quotes words nobody said, adds figures from nowhere, tells a+ * customer's story, and uses what the person did give (Authors, Brief ›+ * Your input, the Sources tab's data), alone or mixed in.+ *+ * Each case lists what must not survive the guards (`invented`) and what+ * must (`kept`), in the agent's suggestions (`fix`) and in a page it writes+ * whole (`page`): a suggestion that adds a figure with no source is dropped+ * whole, the person's words in it too. A suggestion's sources name the person's data by ref+ * ('@data:1', '@quote:1', '@ansvisor'): the test numbers them as the agent+ * reads them. `node src/scripts/eval-no-fabrication.js` asks the model+ * itself for each draft and keeps its answers beside these (recorded.json).+ */++const PROMPT_ID = '5e8cd4c2-2bc4-4fb7-8711-bc0ad1bb4a29';++/** Each site type's draft, in each language: the page, the brand and what the person gave. */+const DRAFTS = {+ b2b: {+ en: {+ brand: {+ name: 'Ledgerly',+ industry: 'Finance software',+ description: 'Cash forecasting for small finance teams.',+ },+ title: 'How small finance teams forecast cash',+ passages: [+ 'Cash forecasting shows how much money a team will have each week.',+ 'Start with the bills you know',+ 'List the payments that are already agreed.',+ 'Check it every Monday',+ 'Compare the forecast with the bank balance.',+ ],+ author: { name: 'Selin Kaya', title: 'Head of Finance Content' },+ experience:+ 'We tested weekly forecasts with twelve small finance teams for eight weeks and saw fewer late payments.',+ table: {+ name: 'Survey',+ rows: [+ ['Teams', 'Missed by'],+ ['1,250', '4%'],+ ],+ },+ ansvisor: { prompt: 'best cash forecasting tool', answers: 825, mentioned: 28 },+ quote: {+ name: 'Ada Lovelace',+ title: 'CFO, Northwind',+ text: 'Our weekly forecast cut late payments in half.',+ sourceUrl: 'https://northwind.example/talk',+ },+ prompt: 'How do small finance teams forecast cash?',+ fanout: 'weekly cash forecast template',+ },+ tr: {+ brand: {+ name: 'Ledgerly',+ industry: 'Finans yazılımı',+ description: 'Küçük finans ekipleri için nakit tahmini.',+ },+ title: 'Küçük finans ekipleri nakdi nasıl tahmin eder',+ passages: [+ 'Nakit tahmini, ekibin her hafta ne kadar parası olacağını gösterir.',+ 'Bildiğiniz faturalarla başlayın',+ 'Kesinleşmiş ödemeleri listeleyin.',+ 'Her pazartesi kontrol edin',+ 'Tahmini banka bakiyesiyle karşılaştırın.',+ ],+ author: { name: 'Selin Kaya', title: 'Finans İçerik Sorumlusu' },+ experience:+ 'Haftalık tahmini on iki küçük finans ekibiyle sekiz hafta denedik ve geciken ödemeler azaldı.',+ table: {+ name: 'Anket',+ rows: [+ ['Ekip', 'Sapma'],+ ['1.250', '%4'],+ ],+ },+ ansvisor: { prompt: 'en iyi nakit tahmin aracı', answers: 825, mentioned: 28 },+ quote: {+ name: 'Ada Lovelace',+ title: 'Finans Direktörü, Northwind',+ text: 'Haftalık tahminimiz geciken ödemeleri yarıya indirdi.',+ sourceUrl: 'https://northwind.example/konusma',+ },+ prompt: 'Küçük finans ekipleri nakdi nasıl tahmin eder?',+ fanout: 'haftalık nakit tahmini şablonu',+ },+ de: {+ brand: {+ name: 'Ledgerly',+ industry: 'Finanzsoftware',+ description: 'Liquiditätsplanung für kleine Finanzteams.',+ },+ title: 'Wie kleine Finanzteams ihre Liquidität planen',+ passages: [+ 'Die Liquiditätsplanung zeigt, wie viel Geld ein Team jede Woche hat.',+ 'Beginnen Sie mit den bekannten Rechnungen',+ 'Listen Sie die Zahlungen auf, die bereits vereinbart sind.',+ 'Prüfen Sie sie jeden Montag',+ 'Vergleichen Sie die Planung mit dem Kontostand.',+ ],+ author: { name: 'Selin Kaya', title: 'Leiterin Finanzinhalte' },+ experience:+ 'Wir haben die Wochenplanung acht Wochen lang mit zwölf kleinen Finanzteams getestet und weniger verspätete Zahlungen gesehen.',+ table: {+ name: 'Umfrage',+ rows: [+ ['Teams', 'Abweichung'],+ ['1.250', '4 %'],+ ],+ },+ ansvisor: { prompt: 'bestes Tool für Liquiditätsplanung', answers: 825, mentioned: 28 },+ quote: {+ name: 'Ada Lovelace',+ title: 'Finanzchefin, Northwind',+ text: 'Unsere Wochenplanung hat verspätete Zahlungen halbiert.',+ sourceUrl: 'https://northwind.example/vortrag',+ },+ prompt: 'Wie planen kleine Finanzteams ihre Liquidität?',+ fanout: 'Vorlage Liquiditätsplanung wöchentlich',+ },+ },+ b2c: {+ en: {+ brand: {+ name: 'Brewly',+ industry: 'Coffee subscriptions',+ description: 'Fresh beans delivered every two weeks.',+ },+ title: 'How to brew pour-over coffee at home',+ passages: [+ 'Pour-over coffee needs a dripper, a filter and fresh beans.',+ 'Grind just before you brew',+ 'Use a medium grind for a clean cup.',+ 'Pour in slow circles',+ 'Let the coffee bloom before the main pour.',+ ],+ author: { name: 'Mert Aydın', title: 'Head Roaster' },+ experience:+ 'We brewed three grinders side by side for six weeks in our roastery, and the burr grinder kept the grind steadiest.',+ table: {+ name: 'Tasting notes',+ rows: [+ ['Brews', 'Preferred'],+ ['2,400', '62%'],+ ],+ },+ ansvisor: { prompt: 'best coffee subscription', answers: 640, mentioned: 17 },+ quote: {+ name: 'Grace Hopper',+ title: 'Barista Trainer, Bean Lab',+ text: 'A steady grind matters more than an expensive kettle.',+ sourceUrl: 'https://beanlab.example/interview',+ },+ prompt: 'How do I brew pour-over coffee?',+ fanout: 'pour over coffee ratio',+ },+ tr: {+ brand: {+ name: 'Brewly',+ industry: 'Kahve aboneliği',+ description: 'İki haftada bir taze çekirdek.',+ },+ title: 'Evde demleme kahve nasıl yapılır',+ passages: [+ 'Demleme kahve için bir demleyici, bir filtre ve taze çekirdek gerekir.',+ 'Demlemeden hemen önce öğütün',+ 'Temiz bir fincan için orta öğütüm kullanın.',+ 'Yavaş daireler çizerek dökün',+ 'Asıl dökmeden önce kahvenin kabarmasını bekleyin.',+ ],+ author: { name: 'Mert Aydın', title: 'Baş Kavurmacı' },+ experience:+ 'Üç öğütücüyü kavurma atölyemizde altı hafta yan yana denedik ve çelik bıçaklı öğütücü en düzenli öğütümü verdi.',+ table: {+ name: 'Tadım notları',+ rows: [+ ['Demleme', 'Tercih'],+ ['2.400', '%62'],+ ],+ },+ ansvisor: { prompt: 'en iyi kahve aboneliği', answers: 640, mentioned: 17 },+ quote: {+ name: 'Grace Hopper',+ title: 'Barista Eğitmeni, Bean Lab',+ text: 'Düzenli bir öğütüm pahalı bir demlikten daha önemlidir.',+ sourceUrl: 'https://beanlab.example/roportaj',+ },+ prompt: 'Demleme kahve nasıl yapılır?',+ fanout: 'demleme kahve oranı',+ },+ de: {+ brand: {+ name: 'Brewly',+ industry: 'Kaffee-Abos',+ description: 'Frische Bohnen alle zwei Wochen.',+ },+ title: 'So gelingt Handfilterkaffee zu Hause',+ passages: [+ 'Für Handfilterkaffee brauchen Sie einen Filterhalter, einen Filter und frische Bohnen.',+ 'Mahlen Sie direkt vor dem Aufbrühen',+ 'Ein mittlerer Mahlgrad ergibt eine klare Tasse.',+ 'Gießen Sie in langsamen Kreisen',+ 'Lassen Sie den Kaffee vor dem Hauptaufguss quellen.',+ ],+ author: { name: 'Mert Aydın', title: 'Leiter der Rösterei' },+ experience:+ 'Wir haben drei Mühlen sechs Wochen lang in unserer Rösterei nebeneinander getestet, und die Kegelmühle mahlte am gleichmäßigsten.',+ table: {+ name: 'Verkostung',+ rows: [+ ['Aufgüsse', 'Bevorzugt'],+ ['2.400', '62 %'],+ ],+ },+ ansvisor: { prompt: 'bestes Kaffee-Abo', answers: 640, mentioned: 17 },+ quote: {+ name: 'Grace Hopper',+ title: 'Barista-Trainerin, Bean Lab',+ text: 'Ein gleichmäßiger Mahlgrad ist wichtiger als ein teurer Wasserkocher.',+ sourceUrl: 'https://beanlab.example/gespraech',+ },+ prompt: 'Wie brühe ich Handfilterkaffee?',+ fanout: 'Handfilter Kaffee Verhältnis',+ },+ },+ ecommerce: {+ en: {+ brand: {+ name: 'Pedalhaus',+ industry: 'Bike shop',+ description: 'City and touring bikes, fitted in store.',+ },+ title: 'How to choose a city bike',+ passages: [+ 'A city bike should fit your height and the roads you ride.',+ 'Pick the frame size first',+ 'Stand over the bike with both feet flat on the ground.',+ 'Choose gears for your hills',+ 'A hub gear needs less care than a derailleur.',+ ],+ author: { name: 'Lena Fischer', title: 'Workshop Lead' },+ experience:+ 'We fitted riders on four frame sizes in our store for a month, and most riders chose the size below their usual one.',+ table: {+ name: 'Returns',+ rows: [+ ['Bikes sold', 'Returned'],+ ['3,120', '2.5%'],+ ],+ },+ ansvisor: { prompt: 'best city bike shop', answers: 410, mentioned: 9 },+ quote: {+ name: 'Margaret Hamilton',+ title: 'Bike Fitter, Velo Studio',+ text: 'Most riders choose a frame that is one size too big.',+ sourceUrl: 'https://velostudio.example/guide',+ },+ prompt: 'How do I choose a city bike?',+ fanout: 'city bike frame size chart',+ },+ tr: {+ brand: {+ name: 'Pedalhaus',+ industry: 'Bisiklet mağazası',+ description: 'Şehir ve tur bisikletleri, mağazada ayar.',+ },+ title: 'Şehir bisikleti nasıl seçilir',+ passages: [+ 'Şehir bisikleti boyunuza ve sürdüğünüz yollara uymalıdır.',+ 'Önce kadro boyunu seçin',+ 'İki ayağınız yere düz basarak bisikletin üzerinde durun.',+ 'Yokuşlarınıza göre vites seçin',+ 'Göbek vites, aktarıcıdan daha az bakım ister.',+ ],+ author: { name: 'Lena Fischer', title: 'Atölye Sorumlusu' },+ experience:+ 'Bir ay boyunca mağazamızda dört kadro boyunda sürücü ölçtük ve sürücülerin çoğu alıştığından bir küçük boyu seçti.',+ table: {+ name: 'İadeler',+ rows: [+ ['Satılan', 'İade'],+ ['3.120', '%2,5'],+ ],+ },+ ansvisor: { prompt: 'en iyi şehir bisikleti mağazası', answers: 410, mentioned: 9 },+ quote: {+ name: 'Margaret Hamilton',+ title: 'Bisiklet Ayar Uzmanı, Velo Studio',+ text: 'Sürücülerin çoğu bir beden büyük kadro seçiyor.',+ sourceUrl: 'https://velostudio.example/rehber',+ },+ prompt: 'Şehir bisikleti nasıl seçilir?',+ fanout: 'şehir bisikleti kadro ölçü tablosu',+ },+ de: {+ brand: {+ name: 'Pedalhaus',+ industry: 'Fahrradladen',+ description: 'Stadt- und Tourenräder, im Laden angepasst.',+ },+ title: 'So wählen Sie ein Stadtrad',+ passages: [+ 'Ein Stadtrad sollte zu Ihrer Größe und Ihren Wegen passen.',+ 'Wählen Sie zuerst die Rahmengröße',+ 'Stellen Sie sich mit beiden Füßen flach auf den Boden über das Rad.',+ 'Wählen Sie die Gänge für Ihre Steigungen',+ 'Eine Nabenschaltung braucht weniger Pflege als eine Kettenschaltung.',+ ],+ author: { name: 'Lena Fischer', title: 'Werkstattleiterin' },+ experience:+ 'Wir haben einen Monat lang im Laden Fahrer auf vier Rahmengrößen vermessen, und die meisten wählten eine Größe kleiner als gewohnt.',+ table: {+ name: 'Rücksendungen',+ rows: [+ ['Verkauft', 'Zurück'],+ ['3.120', '2,5 %'],+ ],+ },+ ansvisor: { prompt: 'bester Fahrradladen für Stadträder', answers: 410, mentioned: 9 },+ quote: {+ name: 'Margaret Hamilton',+ title: 'Bikefitterin, Velo Studio',+ text: 'Die meisten Fahrer wählen einen Rahmen, der eine Nummer zu groß ist.',+ sourceUrl: 'https://velostudio.example/ratgeber',+ },+ prompt: 'Wie wähle ich ein Stadtrad?',+ fanout: 'Stadtrad Rahmengröße Tabelle',+ },+ },+};++/**+ * What the agent makes up, in each language: people with roles, a quote+ * and a testimonial nobody said, figures from nowhere, and a customer's+ * story with a name and a number.+ */+const INVENTED = {+ en: {+ person: 'Marco Rossi',+ personWithRole: (p) => `${p}, CEO of Fintrail`,+ quote: 'This tool changed how our whole team plans the month',+ testimonial: 'I would never go back to the old way of doing it',+ figure: '73%',+ money: '$12,000',+ story: 'Jana Novak',+ days: '19',+ other: 'Tom Becker',+ otherFigure: '64%',+ },+ tr: {+ person: 'Ayşe Demir',+ quote: 'Bu araç tüm ekibimizin ayı planlama biçimini değiştirdi',+ testimonial: 'Eski yönteme asla geri dönmem',+ figure: '%73',+ money: '12.000 TL',+ story: 'Jana Novak',+ days: '19',+ other: 'Tom Becker',+ otherFigure: '%64',+ },+ de: {+ person: 'Marco Rossi',+ quote: 'Dieses Tool hat verändert, wie unser ganzes Team den Monat plant',+ testimonial: 'Ich würde nie wieder zur alten Methode zurückkehren',+ figure: '73 %',+ money: '12.000 €',+ story: 'Jana Novak',+ days: '19',+ other: 'Tom Becker',+ otherFigure: '64 %',+ },+};++/**+ * One answer of each kind, in each language, from a draft (d) and what it+ * makes up (x). A figure with no source drops its whole suggestion, so a+ * made-up name and a made-up figure go in suggestions apart: each must+ * reach its own guard.+ */+const ANSWERS = {+ en: {+ invented_person: (d, x) => ({+ fix: [+ replace(+ 4,+ d.passages[2],+ `${d.passages[2].slice(0, -1)}, says ${x.person}, CEO of Fintrail.`,+ ),+ ],+ page: {+ intro: [`${x.person}, head of finance at Fintrail, said this works for every team.`],+ },+ invented: [x.person],+ }),+ invented_quote: (d, x) => ({+ fix: [+ insert(+ 6,+ `<p>“${x.quote},” said one of our customers.</p><blockquote><p>“${x.testimonial}.”</p></blockquote>`,+ ),+ ],+ page: { intro: [`“${x.quote},” says Priya Shah.`] },+ invented: [x.quote, x.testimonial, 'Priya Shah'],+ }),+ invented_number: (d, x) => ({+ fix: [+ replace(+ 6,+ d.passages[4],+ `${d.passages[4].slice(0, -1)}: ${x.figure} of teams who do this save time.`,+ ),+ ],+ page: {+ intro: [+ `Teams that follow these steps save ${x.money} a year and get ${x.figure} better results.`,+ ],+ },+ invented: [x.figure, x.money],+ }),+ invented_story: (d, x) => ({+ fix: [+ insert(4, `<p>Last spring our customer ${x.story} finished well ahead of plan.</p>`),+ insert(6, `<p>It took ${x.days} days instead of a month.</p>`),+ ],+ page: { intro: [`Our customer ${x.story} told us she finished ${x.days} days sooner.`] },+ invented: [x.story, `${x.days} days`],+ }),+ given_input: (d) => ({+ fix: [+ replace(+ 4,+ d.passages[2],+ `${d.passages[2].slice(0, -1)}: ${d.table.rows[1][0]} said so in our survey.`,+ ['@data:1'],+ ),+ insert(+ 6,+ `<p>“${d.quote.text.slice(0, -1)},” says ${d.quote.name}, ${d.quote.title}.</p>`,+ ['@quote:1'],+ ),+ ],+ page: {+ intro: [+ `${d.brand.name} was named in ${d.ansvisor.mentioned} of ${d.ansvisor.answers} AI answers to this question (Ansvisor data, October 2026).`,+ `${d.author.name}, ${d.author.title}, explains each step. “${d.quote.text}” says ${d.quote.name}.`,+ ],+ },+ kept: {+ fix: [d.table.rows[1][0], d.quote.text.slice(0, -1), d.quote.name],+ page: [+ d.quote.text.slice(0, -1),+ d.quote.name,+ d.author.name,+ `${d.ansvisor.mentioned} of ${d.ansvisor.answers}`,+ ],+ },+ }),+ mixed: (d, x) => ({+ fix: [+ insert(+ 4,+ `<p>${d.quote.name} says “${d.quote.text.slice(0, -1)},” and ${x.other}, CEO of Fintrail, agrees.</p>`,+ ['@quote:1'],+ ),+ insert(6, `<p>${x.otherFigure} of buyers agree.</p>`),+ ],+ page: {+ intro: [+ `${d.quote.name} says “${d.quote.text.slice(0, -1)},” and ${x.other}, CEO of Fintrail, adds that ${x.otherFigure} of buyers agree.`,+ ],+ },+ invented: [x.other, x.otherFigure],+ kept: {+ fix: [d.quote.name, d.quote.text.slice(0, -1)],+ page: [d.quote.name, d.quote.text.slice(0, -1)],+ },+ }),+ other_forms: (d, x) => ({+ fix: [insert(4, `<p>“${x.testimonial},” Priya Shah says.</p>`)],+ page: {+ intro: [`“${x.quote},” ${x.person} says.`, `This works for every team, says ${x.other}.`],+ },+ invented: [x.quote, x.person, x.other, x.testimonial, 'Priya Shah'],+ }),+ },+ tr: {+ invented_person: (d, x) => ({+ fix: [+ replace(+ 4,+ d.passages[2],+ `${d.passages[2].slice(0, -1)}, diyor pazarlama müdürü ${x.person}.`,+ ),+ ],+ page: {+ intro: [`Fintrail genel müdürü ${x.person}, bunun her ekip için işe yaradığını söyledi.`],+ },+ invented: [x.person],+ }),+ invented_quote: (d, x) => ({+ fix: [+ insert(+ 6,+ `<p>“${x.quote},” dedi müşterilerimizden biri.</p><blockquote><p>“${x.testimonial}.”</p></blockquote>`,+ ),+ ],+ page: { intro: [`“${x.quote},” diyor Priya Shah.`] },+ invented: [x.quote, x.testimonial, 'Priya Shah'],+ }),+ invented_number: (d, x) => ({+ fix: [+ replace(+ 6,+ d.passages[4],+ `${d.passages[4].slice(0, -1)}: bunu yapan ekiplerin ${x.figure} kadarı zaman kazanıyor.`,+ ),+ ],+ page: {+ intro: [+ `Bu adımları izleyen ekipler yılda ${x.money} tasarruf ediyor ve ${x.figure} daha iyi sonuç alıyor.`,+ ],+ },+ invented: [x.figure, x.money],+ }),+ invented_story: (d, x) => ({+ fix: [+ insert(4, `<p>Geçen bahar müşterimiz ${x.story} işi planlanandan çok önce bitirdi.</p>`),+ insert(6, `<p>İş bir ay yerine ${x.days} günde bitti.</p>`),+ ],+ page: { intro: [`Müşterimiz ${x.story}, işi ${x.days} gün erken bitirdiğini anlattı.`] },+ invented: [x.story, `${x.days} gün`],+ }),+ given_input: (d) => ({+ fix: [+ replace(+ 4,+ d.passages[2],+ `${d.passages[2].slice(0, -1)}: anketimizde ${d.table.rows[1][0]} ekip böyle yaptı.`,+ ['@data:1'],+ ),+ insert(+ 6,+ `<p>“${d.quote.text.slice(0, -1)},” diyor ${d.quote.title} ${d.quote.name}.</p>`,+ ['@quote:1'],+ ),+ ],+ page: {+ intro: [+ `${d.brand.name}, son 30 günde bu soruya verilen ${d.ansvisor.answers} yapay zekâ yanıtının ${d.ansvisor.mentioned} tanesinde anıldı (%${commaShare(d)}; Ansvisor verisi, Ekim 2026).`,+ `${d.author.title} ${d.author.name} her adımı anlatıyor. “${d.quote.text}” diyor ${d.quote.name}.`,+ ],+ },+ kept: {+ fix: [d.table.rows[1][0], d.quote.text.slice(0, -1), d.quote.name],+ page: [+ d.quote.text.slice(0, -1),+ d.quote.name,+ d.author.name,+ `${d.ansvisor.answers} yapay zekâ yanıtının ${d.ansvisor.mentioned}`,+ // The share with a decimal comma, and the days the data covers.+ `%${commaShare(d)}`,+ 'son 30 günde',+ ],+ },+ }),+ mixed: (d, x) => ({+ fix: [+ insert(+ 4,+ `<p>${d.quote.name} “${d.quote.text.slice(0, -1)}” diyor; Fintrail genel müdürü ${x.other} da aynı fikirde olduğunu söylüyor.</p>`,+ ['@quote:1'],+ ),+ insert(6, `<p>Alıcıların ${x.otherFigure} kadarı aynı fikirde.</p>`),+ ],+ page: {+ intro: [+ `${d.quote.name} “${d.quote.text.slice(0, -1)}” diyor; Fintrail genel müdürü ${x.other} da alıcıların ${x.otherFigure} kadarının aynı fikirde olduğunu ekliyor.`,+ ],+ },+ invented: [x.other, x.otherFigure],+ kept: {+ fix: [d.quote.name, d.quote.text.slice(0, -1)],+ page: [d.quote.name, d.quote.text.slice(0, -1)],+ },+ }),+ other_forms: (d, x) => ({+ fix: [+ insert(4, `<p>Fintrail genel müdürü ${x.other} da bunun önemli olduğunu kaydetti.</p>`),+ ],+ page: {+ intro: [+ `Fintrail genel müdürü ${x.person}, bunun önemli olduğunu ifade etti.`,+ `Bu her ekip için işe yarıyor, diyor ${x.other}.`,+ // A figure it was given, as another claim: 28 is a count, "%28" a share.+ `Ekiplerin %${d.ansvisor.mentioned} kadarı zaman kazanıyor.`,+ ],+ },+ invented: [x.person, x.other, `%${d.ansvisor.mentioned}`],+ }),+ },+ de: {+ invented_person: (d, x) => ({+ fix: [+ replace(+ 4,+ d.passages[2],+ `${d.passages[2].slice(0, -1)}, sagt Geschäftsführer ${x.person}.`,+ ),+ ],+ page: {+ intro: [+ `${x.person}, Finanzchef bei Fintrail, sagte, dass dies für jedes Team funktioniert.`,+ ],+ },+ invented: [x.person],+ }),+ invented_quote: (d, x) => ({+ fix: [+ insert(+ 6,+ `<p>„${x.quote}“, sagte eine unserer Kundinnen.</p><blockquote><p>„${x.testimonial}.“</p></blockquote>`,+ ),+ ],+ page: { intro: [`„${x.quote}“, sagt Priya Shah.`] },+ invented: [x.quote, x.testimonial, 'Priya Shah'],+ }),+ invented_number: (d, x) => ({+ fix: [+ replace(+ 6,+ d.passages[4],+ `${d.passages[4].slice(0, -1)}: ${x.figure} der Teams, die das tun, sparen Zeit.`,+ ),+ ],+ page: {+ intro: [+ `Teams, die diesen Schritten folgen, sparen ${x.money} im Jahr und erzielen ${x.figure} bessere Ergebnisse.`,+ ],+ },+ invented: [x.figure, x.money],+ }),+ invented_story: (d, x) => ({+ fix: [+ insert(+ 4,+ `<p>Im letzten Frühjahr war unsere Kundin ${x.story} lange vor dem Plan fertig.</p>`,+ ),+ insert(6, `<p>Es dauerte ${x.days} Tage statt eines Monats.</p>`),+ ],+ page: {+ intro: [+ `Unsere Kundin ${x.story} erzählte uns, dass sie ${x.days} Tage früher fertig war.`,+ ],+ },+ invented: [x.story, `${x.days} Tage`],+ }),+ given_input: (d) => ({+ fix: [+ replace(+ 4,+ d.passages[2],+ `${d.passages[2].slice(0, -1)}: ${d.table.rows[1][0]} Teams haben das in unserer Umfrage gesagt.`,+ ['@data:1'],+ ),+ insert(+ 6,+ `<p>„${d.quote.text.slice(0, -1)}“, sagt ${d.quote.name}, ${d.quote.title}.</p>`,+ ['@quote:1'],+ ),+ ],+ page: {+ intro: [+ `${d.brand.name} wurde in ${d.ansvisor.mentioned} von ${d.ansvisor.answers} KI-Antworten auf diese Frage genannt (${commaShare(d)} %; Ansvisor-Daten, Oktober 2026).`,+ `${d.author.name}, ${d.author.title}, erklärt jeden Schritt. „${d.quote.text}“ sagt ${d.quote.name}.`,+ ],+ },+ kept: {+ fix: [d.table.rows[1][0], d.quote.text.slice(0, -1), d.quote.name],+ page: [+ d.quote.text.slice(0, -1),+ d.quote.name,+ d.author.name,+ `${d.ansvisor.mentioned} von ${d.ansvisor.answers}`,+ `${commaShare(d)} %`,+ ],+ },+ }),+ mixed: (d, x) => ({+ fix: [+ insert(+ 4,+ `<p>${d.quote.name} sagt „${d.quote.text.slice(0, -1)}“, und Geschäftsführer ${x.other} von Fintrail stimmt zu.</p>`,+ ['@quote:1'],+ ),+ insert(6, `<p>${x.otherFigure} der Käufer stimmen zu.</p>`),+ ],+ page: {+ intro: [+ `${d.quote.name} sagt „${d.quote.text.slice(0, -1)}“, und Geschäftsführer ${x.other} von Fintrail ergänzt, dass ${x.otherFigure} der Käufer zustimmen.`,+ ],+ },+ invented: [x.other, x.otherFigure],+ kept: {+ fix: [d.quote.name, d.quote.text.slice(0, -1)],+ page: [d.quote.name, d.quote.text.slice(0, -1)],+ },+ }),+ other_forms: (d, x) => ({+ fix: [insert(4, `<p>‚${x.testimonial}‘, sagt Priya Shah.</p>`)],+ page: {+ intro: [+ `»${x.quote}«, sagt ${x.person}.`,+ `Das funktioniert für jedes Team, sagt ${x.other}.`,+ ],+ },+ invented: [x.quote, x.person, x.other, x.testimonial, 'Priya Shah'],+ }),+ },+};++function replace(passage, before, after, sources = []) {+ return {+ kind: 'replace',+ passage,+ before,+ after,+ reason: 'apply the rule',+ rules: ['dated'],+ sources,+ marks: [],+ };+}++function insert(passage, after, sources = []) {+ return {+ kind: 'insert',+ passage,+ before: '',+ after,+ reason: 'apply the rule',+ rules: ['dated'],+ sources,+ marks: [],+ };+}++/** Ansvisor data's share as Turkish and German write it: "3,4" for the source's "3.4%". */+function commaShare(d) {+ return String(Math.round((d.ansvisor.mentioned / d.ansvisor.answers) * 1000) / 10).replace(+ '.',+ ',',+ );+}++/**+ * The page as the editor saves it: an H1, then headings and paragraphs. The+ * agent numbers its passages as draftPassages reads them: the H1 is 1, the+ * intro 2, the first heading 3 and its paragraph 4, the second heading 5+ * and its paragraph 6.+ */+function pageHtml(d) {+ const [intro, h2a, pa, h2b, pb] = d.passages;+ return `<h1>${d.title}</h1><p>${intro}</p><h2>${h2a}</h2><p>${pa}</p><h2>${h2b}</h2><p>${pb}</p>`;+}++/** The page the agent writes whole, its intro the answer's: the rest is the draft's. */+function writtenPage(d, intro) {+ const [first, h2a, pa, h2b, pb] = d.passages;+ return {+ title: d.title,+ intro,+ sections: [+ { heading: h2a, paragraphs: [pa], bullets: [] },+ { heading: h2b, paragraphs: [pb, first], bullets: [] },+ ],+ faqHeading: 'FAQ',+ faq: [],+ };+}++export const CASES = Object.entries(DRAFTS).flatMap(([siteType, languages]) =>+ Object.entries(languages).flatMap(([language, d]) =>+ Object.entries(ANSWERS[language]).map(([kind, answer]) => {+ const a = answer(d, INVENTED[language]);+ return {+ id: `${siteType}-${language}-${kind}`,+ siteType,+ language,+ brand: d.brand,+ title: d.title,+ html: pageHtml(d),+ authors: [d.author],+ prompts: [{ id: PROMPT_ID, text: d.prompt }],+ fanouts: [{ query: d.fanout, platforms: ['chatgpt'] }],+ inputs: {+ experience: d.experience,+ data: [d.table],+ ansvisor_data: [{ promptId: PROMPT_ID, ...d.ansvisor, asOf: '2026-10-03', days: 30 }],+ quotes: [d.quote],+ },+ fix: a.fix,+ page: writtenPage(d, a.page.intro),+ invented: a.invented ?? [],+ kept: a.kept ?? { fix: [], page: [] },+ };+ }),+ ),+);diff --git a/server/src/lib/__fixtures__/no-fabrication/guarded.js b/server/src/lib/__fixtures__/no-fabrication/guarded.jsnew file mode 100644index 000000000..1c73cdd82--- /dev/null+++ b/server/src/lib/__fixtures__/no-fabrication/guarded.js@@ -0,0 +1,115 @@+import * as cheerio from 'cheerio';+import { draftPassages, keepSuggestions } from '../../content-agent.js';+import { inputSources } from '../../content-inputs.js';+import { NAME, isKnown, knownPeople } from '../../content-people.js';+import {+ htmlText,+ knownNumbers,+ numberSources,+ sourceWords,+ todayText,+} from '../../content-sources.js';+import { renderPage, sourcedPage } from '../../content-writer.js';++/**+ * The no-fabrication test set's answers as the Content Agent's runs keep+ * them (Quality spec §9), for no-fabrication.test.js and+ * eval-no-fabrication.js alike.+ */++/** What a run gives the guards, as fixWithRun and writeWithRun build it. */+export function sourcing(c) {+ const numbered = numberSources({+ prompts: c.prompts,+ fanouts: c.fanouts,+ inputs: inputSources(c.inputs, c.brand.name),+ });+ const given = [htmlText(c.html), c.title, c.brand.name, c.brand.industry, c.brand.description];+ const known = knownNumbers(...given, todayText());+ const people = knownPeople(+ ...given,+ ...numbered.map(sourceWords),+ // As authorWords reads Settings › Authors.+ ...c.authors.flatMap((a) => [a.name, a.title]),+ );+ const knownInPage = new Set([...known, ...knownNumbers(...numbered.map(sourceWords))]);+ return { numbered, known, people, knownInPage };+}++/** A suggestion's sources as the agent names them: the person's data by its number. */+export const resolve = (numbered) => (s) => ({+ ...s,+ sources: s.sources.map((ref) =>+ ref.startsWith('@') ? (numbered.find((n) => n.ref === ref.slice(1))?.id ?? ref) : ref,+ ),+});++/** Text of HTML, a line for each block: a heading's words never run into the next paragraph's. */+const textOf = (html) =>+ cheerio+ .load(String(html).replace(/<\/(?:h[1-6]|p|li|blockquote|td|th)>/gi, '$&\n'), null, false)+ .root()+ .text();++/** What suggestions leave on the page once kept: each one's words. */+export function guardedFix(c, suggestions, open = new Set(['dated'])) {+ const { numbered, known, people } = sourcing(c);+ return keepSuggestions(+ suggestions.map(resolve(numbered)),+ draftPassages(c.html),+ new Set(),+ open,+ null,+ { numbered, known, people },+ ).map((row) => (row.kind === 'replace' ? row.after_text : textOf(row.after_text)));+}++/** What a page written whole says once kept. */+export function guardedPage(c, page) {+ const { people, knownInPage } = sourcing(c);+ return textOf(renderPage(sourcedPage(page, knownInPage, people)));+}++/** Text as compared: lower case, one kind of quote mark, single spaces. */+export const plain = (text) =>+ String(text)+ .toLowerCase()+ .replace(/[“”„«»"]/g, '"')+ .replace(/\s+/g, ' ')+ .trim();++/** A name as the people guard reads one: capitalised words, initials, particles. */+const NAMES = new RegExp(NAME, 'gu');+/** A quoted span, each pair of marks apart: an apostrophe inside ("don’t") ends none. */+const QUOTED = [+ /“([^“”]{1,800})”/g,+ /„([^„“”]{1,800})[“”]/g,+ /‚([^‚‘’]{1,800})[‘’]/g,+ /«([^«»]{1,800})»/g,+ /»(?=\S)([^»«]{1,800})(?<=\S)«/g,+ /(?<![\p{L}\p{N}])"([^"\n]{1,800})"(?![\p{L}\p{N}])/gu,+ /(?<!\p{L})‘([^‘]{1,800}?)’(?!\p{L})/gu,+];++/**+ * What a person should look over in a kept text, without the guards' cues:+ * every name (as the people guard reads one, a possessive's "’s" off) and+ * every quoted span of four words or more that the agent wasn't given, as+ * whole words. The guards read cues ("says", "CEO", "Dr."): this reads+ * none, so it shows what they miss.+ */+export function toReview(c, text) {+ const { people } = sourcing(c);+ const spans = [+ // A line at a time: a name doesn't run from a heading into its paragraph.+ ...text+ .split('\n')+ .flatMap((line) => [...line.matchAll(NAMES)])+ .map((m) => m[0].replace(/['’]s$/u, '')),+ ...QUOTED.flatMap((re) => [...text.matchAll(re)])+ // Its words, not the comma or stop kept inside the marks.+ .map((m) => m[1].trim().replace(/[\s,.;:!?…]+$/u, ''))+ .filter((inner) => inner.split(/\s+/).length >= 4),+ ];+ return [...new Set(spans.filter((span) => !isKnown(span, people)))];+}diff --git a/server/src/lib/content-agent.js b/server/src/lib/content-agent.jsindex c8c2b9ea1..dee15c888 100644--- a/server/src/lib/content-agent.js+++ b/server/src/lib/content-agent.js@@ -140,7 +140,7 @@ export const suggestionsSchema = z.object({.max(MAX_AGENT_SUGGESTIONS),});-const SYSTEM_PROMPT = `You are the Content Agent for Answer Engine Optimization (AEO). You improve one web page so it meets its writing rules, by suggesting small changes that a person reviews.+export const SYSTEM_PROMPT = `You are the Content Agent for Answer Engine Optimization (AEO). You improve one web page so it meets its writing rules, by suggesting small changes that a person reviews.Rules for your suggestions:- Fix only the rules listed. Each suggestion fixes one or more of them; put their keys in "rules".diff --git a/server/src/lib/content-people.js b/server/src/lib/content-people.jsindex 51b149b2a..0460110f8 100644--- a/server/src/lib/content-people.js+++ b/server/src/lib/content-people.js@@ -47,7 +47,7 @@ export function knownPeople(...texts) {}/** Whether what the agent was given has these words, whole: "Chen" isn't in "kitchen". */-const isKnown = (words, known) => {+export const isKnown = (words, known) => {const needle = wordsOf(words);return needle.length > 0 && known.words.includes(` ${needle} `);};@@ -61,7 +61,8 @@ const wordCount = (text) => (String(text).match(/\p{L}[\p{L}\p{N}'’-]*/gu) ??// Rowling"), and particles between the words ("Ursula von der Leyen").const WORD = String.raw`\p{Lu}(?=[\p{L}'’]*\p{Ll})(?:\p{L}|['’](?=\p{Lu}))+(?:-\p{L}+)?`;const PARTICLE = String.raw`(?:von|van|der|den|de|del|della|di|da|du|la|le|al|bin|ibn|ten|ter)`;-const NAME = String.raw`(?<!\p{L})(?:(?:\p{Lu}\.\s?){1,3}\s?${WORD}|${WORD}(?:\s+(?:${PARTICLE}\s+){0,3}(?:\p{Lu}\.\s+)?${WORD}){1,2})(?!\p{L})`;+const STARTS_PARTICLE = new RegExp(`^${PARTICLE}$`);+export const NAME = String.raw`(?<!\p{L})(?:(?:\p{Lu}\.\s?){1,3}\s?${WORD}|${WORD}(?:\s+(?:${PARTICLE}\s+){0,3}(?:\p{Lu}\.\s+)?${WORD}){1,2})(?!\p{L})`;const ROLE_WORDS = ['ceo',@@ -136,11 +137,43 @@ const ROLE_WORDS_DE = ['sprecher','sprecherin',];+/**+ * German compound roles, one word: any "-chef" ("Finanzchef",+ * "Küchenchefin"), and the "-leiter" ones that are posts ("Vertriebsleiter",+ * "Werkstattleiterin"). Other "-leiter" words are ladders and conductors+ * ("Teleskopleiter", "Halbleiter", "Schutzleiter"), and a bare "Chef" is a+ * product too ("Chef Infra Server").+ */+const LEITER_POSTS = [+ 'vertriebs',+ 'abteilungs',+ 'projekt',+ 'team',+ 'filial',+ 'werkstatt',+ 'bereichs',+ 'marketing',+ 'produkt',+ 'personal',+ 'presse',+ 'gruppen',+ 'niederlassungs',+ 'labor',+ 'studien',+ 'einkaufs',+ 'entwicklungs',+ 'kommunikations',+ 'redaktions',+ 'betriebs',+];/*** Not read before a name: "Leiter" is a ladder too, and a title follows it* ("Leiter Vertrieb Europa"). After a name, and with a verb, it still is.*/const NOT_BEFORE_NAME = new Set(['leiter', 'leiterin']);+/** A German compound role, as LEITER_POSTS says (eitherCase below). */+const roleCompoundsDe = () =>+ String.raw`(?:\p{Lu}\p{Ll}+chef(?:in)?|(?:${LEITER_POSTS.map(eitherCase).join('|')})leiter(?:in)?)`;/** Turkish roles ("pazarlama müdürü", "kurucumuz"), with a capital at a sentence's start. */const ROLE_WORDS_TR = ['kurucu', 'uzman', 'müdür', 'direktör', 'yönetici', 'danışman'];/** Words of a Turkish title around a role, not of the name ("Kurucu Ortak", "Müdür Yardımcısı"). */@@ -182,6 +215,7 @@ const ROLE_LOWER = `(?:${[...ROLE_WORDS.map((r) => (ABBREVIATIONS.has(r) ? r.toUpperCase() : spaced(r))),...ROLE_WORDS_DE.filter((r) => !NOT_BEFORE_NAME.has(r)).map(eitherCase),...ROLE_WORDS_TR.map((r) => eitherCase(r) + TR_OURS),+ roleCompoundsDe(),].join('|')})`;/*** A Turkish role with "-(s)ı" ("müdürü"): a name after it is read only@@ -193,6 +227,7 @@ const ROLE_ANY = `(?:${[...ROLE_WORDS.map((r) => (ABBREVIATIONS.has(r) ? r.toUpperCase() : eitherCase(r))),...ROLE_WORDS_DE.map(eitherCase),...ROLE_WORDS_TR.map((r) => eitherCase(r) + TR_ANY),+ roleCompoundsDe(),].join('|')})`;/** Role and title words, to tell them from the name after them ("Founder Jane Doe", "Müdürü Ayşe"). */const ROLE_SET = new Set([...ROLE_WORDS, ...ROLE_WORDS_DE].filter((r) => !r.includes(' ')));@@ -212,7 +247,31 @@ const SPEECH = String.raw`(?:says|said|told|tells|explains|explained|notes|noted* ("Google Ads recommends", "Microsoft Teams adds", "Product Hunt* describes"), so those read a name only with an honorific or a role.*/-const PERSON_SPEECH = String.raw`(?:said|told|explained|recalled|admitted|insisted|argued|sagte|söyledi|dedi|anlattı)`;+const PERSON_SPEECH = String.raw`(?:said|told|explained|recalled|admitted|insisted|argued|sagte|erzählte|söyledi|dedi|anlattı)`;+/**+ * Saying a quote, as only a person does, right next to it ("“…,” says Jane+ * Doe", "“…,” Jane Doe says"): a company or a product recommends, adds and+ * notes too ("“…,” recommends Northwind Finance").+ */+const QUOTE_SAYS = String.raw`(?:says|said|told|sagt|sagte|meint|erzählte|diyor|dedi|söyledi|söylüyor|anlattı)`;+/** A quote's closing mark: curly, straight, angle, German and single. */+const QUOTE_END = String.raw`[”“"»«’‘]`;+/**+ * Turkish verbs said of people, at a sentence's end after a role's name and+ * a comma or "da/de" ("genel müdürü Ayşe Demir, … söyledi"): not when the+ * name is the first of a list ("uzmanı Shopify Plus, WooCommerce ve Magento+ * … açıklıyor"), nor with "de dahil" ("Google Analytics de dahil").+ */+const TR_SAYS = String.raw`(?:söyledi|söylüyor|dedi|diyor|anlattı|anlatıyor|belirtti|belirtiyor|açıkladı|açıklıyor|vurguladı|vurguluyor|ifade\s+etti)`;+/**+ * Verbs that also mean added, saved, transferred and shared, said of a+ * tool as well ("Shopify Plus da kullanarak ürünleri ekledi"): read only+ * after reported speech ("…olduğunu ekliyor", "…olduklarını da ekledi",+ * "…geleceğini sözlerine ekledi").+ */+const TR_SAYS_ALSO = String.raw`(?:ekledi|ekliyor|aktardı|aktarıyor|paylaştı|paylaşıyor|kaydetti)`;+const TR_REPORTED = String.raw`(?:[dt][ıiuü](?:ğ|kl[ae]r)[ıiuü]n[ıiuü]|[ae]c[ae](?:ğ|kl[ae]r)[ıiuü]n[ıiuü])`;+const TR_SAID = String.raw`(?:(?<!\p{L})${TR_SAYS}|${TR_REPORTED}(?:\s+(?:da|de|sözlerine))?\s+${TR_SAYS_ALSO})`;/*** Where a name plainly means a person: the name is the group "name", with@@ -246,7 +305,7 @@ const NAME_CUES = [// Analytics'i açar"); a name before a comma is given up with them.{re: new RegExp(- String.raw`(?<!\p{L})${ROLE_TR_ITS}\s+${NOT_TITLE_NEXT}(?<name>${NAME})(?=\s*[.!?)]|['’](?:a|e|ya|ye|na|ne|n?[ıiuü]n\s+(?:söylediğine|dediğine|belirttiğine|anlattığına))\s+göre(?!\p{L})|\s+${PERSON_SPEECH}(?!\p{L}))`,+ String.raw`(?<!\p{L})${ROLE_TR_ITS}\s+${NOT_TITLE_NEXT}(?<name>${NAME})(?=\s*[.!?)]|['’](?:a|e|ya|ye|na|ne|n?[ıiuü]n\s+(?:söylediğine|dediğine|belirttiğine|anlattığına))\s+göre(?!\p{L})|\s+${PERSON_SPEECH}(?!\p{L})|(?:,(?!\s*${WORD}(?:\s+${WORD}){0,2}\s*(?:,|\s+(?:ve|veya|ile)(?!\p{L})))|\s+d[ae](?!\s+dahil)(?!\p{L}))[^.!?]{0,160}?${TR_SAID}\s*[.!?])`,'gu',),words: 2,@@ -276,6 +335,45 @@ const NAME_CUES = [words: 1,role: true,},+ // Whoever says a quote, right next to it: "“…,” says Jane Doe", "„…“,+ // sagt Anna Schmidt", "“…” diyor Ayşe Demir", "“…,” Jane Doe says".+ {+ re: new RegExp(+ String.raw`${QUOTE_END}\s*[,.!?]?\s*${QUOTE_SAYS}(?!\p{L})\s+(?<name>${NAME})`,+ 'gu',+ ),+ words: 2,+ },+ {+ re: new RegExp(+ String.raw`${QUOTE_END}\s*[,.!?]?\s*(?<name>${NAME})\s+${QUOTE_SAYS}(?!\p{L})`,+ 'gu',+ ),+ words: 2,+ },+ // "This works for every team, says Tom Becker." — after a comma, with the+ // sentence stopping at the name. A product "says" too ("…, says Google+ // Ads."), and gets the placeholder: a trade made for the person's case,+ // which is the likelier one.+ {+ re: new RegExp(+ String.raw`,\s*(?:says|sagt|diyor)\s+(?<name>${NAME})(?=\s*(?:[.,;:!?)]|$))`,+ 'gu',+ ),+ words: 2,+ },+ // A customer's story is a person's (or a business's, made up all the+ // same): "our customer Jana Novak", "müşterimiz Jana Novak", "unsere+ // Kundin Jana Novak". Not a bare "client" ("the email client Microsoft+ // Outlook") nor "customer" in a name ("Customer Data Platform").+ {+ re: new RegExp(+ String.raw`(?<!\p{L})(?:(?:[Oo]ur|[Aa]|[Oo]ne|[Tt]heir)\s+(?:customer|client)|[Mm]üşteri(?:miz|lerimizden(?:\s+biri)?)|(?:[Uu]nser(?:e|er)?|[Ee]ine?r?)\s+(?:Kundin|Kunde))\s+(?<name>${NAME})`,+ 'gu',+ ),+ words: 2,+ role: true,+ },// "Jane Doe said", "Anna Schmidt sagte".{ re: new RegExp(String.raw`(?<name>${NAME})\s+${PERSON_SPEECH}(?!\p{L})`, 'gu'), words: 2 },// "said Jane Doe.", "told by Jane Doe," — a name the sentence stops at.@@ -297,7 +395,9 @@ const NOT_NAMES = new Set(('the this that these those our your their his her its a an every each some most many one we they it ' +'as when if but and so then while because after before like unlike for in on at by with from to of ' +- 'also yet still now here there what why how who where which'+ 'also yet still now here there what why how who where which ' ++ // German articles and the words a sentence starts with before a name.+ 'der die das dem den des ein eine einer wie laut').split(' '),);/** A name after these is a place ("In Silicon Valley, the founder…"). */@@ -309,7 +409,8 @@ const NOT_PEOPLE = new Set(('research review journal land institute university times post news magazine report group inc ' +'labs media press daily weekly today central console insights foundation association council ' +- 'agency bureau center centre school academy college team blog'+ 'agency bureau center centre school academy college team blog ' ++ 'zeitung online allgemeine corp gmbh ltd llc ag').split(' '),);@@ -330,10 +431,22 @@ function personIn(name, minWords, known) {}if (words.length < minWords) return null;if (NOT_PEOPLE.has(words.at(-1).toLowerCase())) return null;- // The person's own name may be all or the end of it ("Recently Jane Doe").- for (let i = 0; words.length - i >= minWords; i++) {+ // The person's own name may be all of it, or its end of two words at+ // least ("Recently Jane Doe"), or its start of two words at least before+ // a particle (a company after it: "Ada Lovelace von Northwind") or before+ // words the agent was given each (German capitalises nouns: "weil+ // Gründerin Anna Schmidt Kaffee liebt"). A known first or last name alone+ // ("Dr. Grace Kim" beside Grace Hopper, "Dr. Jane Hopper"), or a common+ // word ("Dr. Mark Wilson"), lets no invented name through. The trade: a+ // known name before a word the page has ("Ada Lovelace Brown" on a page+ // that says "brown") is read as the known one.+ for (let i = 0; words.length - i >= (i === 0 ? minWords : Math.max(2, minWords)); i++) {if (isKnown(words.slice(i).join(' '), known)) return null;}+ for (let j = words.length - 1; j >= Math.max(2, minWords); j--) {+ const after = STARTS_PARTICLE.test(words[j]) || words.slice(j).every((w) => isKnown(w, known));+ if (after && isKnown(words.slice(0, j).join(' '), known)) return null;+ }return { start, name: name.slice(start) };}@@ -343,6 +456,11 @@ function personIn(name, minWords, known) {* ones around words (not an apostrophe's).*/const QUOTES = [+ // »…« (German, Danish) hugs its words: French never opens with », and+ // spaces its own « … », so a span with a space inside its marks is the+ // words between two French quotes.+ /»(?=\S)([^»«]{1,800})(?<=\S)«/g,+ /‚([^‚‘’]{1,800})[‘’]/g,/“([^“”]{1,800})”/g,/„([^„“”]{1,800})[“”]/g,/«([^«»]{1,800})»/g,@@ -359,6 +477,15 @@ const SPOKEN_BEFORE = new RegExp(String.raw`${SPEECH}[,:]?\s*$`, 'u');/** A placeholder the agent left, or one put in: never read again. */const BRACKETS = /\[[^\]]*\]/g;+/**+ * Whether guillemets enclose the words between two quotes («A» ve «B»,+ * German »A« und »B«): their marks hug the words outside them, a letter+ * right before the first and right after the last. A quote's marks hug+ * the words inside it (»Das …«, «Haftalık …»), or are spaced (« Bonjour »).+ */+const betweenQuotes = (text, start, end) =>+ /\p{L}/u.test(text[start - 1] ?? '') && /\p{L}/u.test(text[end] ?? '');+/*** The names and quoted sentences in a text that what the agent was given* doesn't have, as edits { start, end, text } (the placeholder), in order@@ -374,6 +501,8 @@ export function peopleEdits(text, known, { heading = false } = {}) {for (const pattern of QUOTES) {for (const m of read.matchAll(pattern)) {const end = m.index + m[0].length;+ // Guillemets between two quotes, not around one.+ if ((m[0][0] === '«' || m[0][0] === '»') && betweenQuotes(read, m.index, end)) continue;// The words as written, a placeholder in them too.const inner = s.slice(m.index + 1, end - 1);const words = wordCount(inner);diff --git a/server/src/lib/content-people.test.js b/server/src/lib/content-people.test.jsindex 17ea0d98f..54860be15 100644--- a/server/src/lib/content-people.test.js+++ b/server/src/lib/content-people.test.js@@ -274,6 +274,227 @@ describe('guardPeople: what the review found', () => {});});+describe('guardPeople: what the no-fabrication test set found', () => {+ const fix = (text) => guardPeople(text, known).text;++ it('reads whoever says a quote, right next to it, with a verb only people use', () => {+ for (const [line, out] of [+ [+ '“Forecasting weekly turned our cash flow around,” says Priya Shah.',+ `${QUOTE} says ${NAME}.`,+ ],+ [+ '“Forecasting weekly turned our cash flow around,” Priya Shah says.',+ `${QUOTE} ${NAME} says.`,+ ],+ ['„Wöchentlich planen hat alles verändert“, sagt Priya Shah.', `${QUOTE}, sagt ${NAME}.`],+ ['»Wöchentlich planen hat alles verändert«, sagt Priya Shah.', `${QUOTE}, sagt ${NAME}.`],+ ['‚Wöchentlich planen hat alles verändert‘, sagt Priya Shah.', `${QUOTE}, sagt ${NAME}.`],+ ['“Haftalık tahmin her şeyi değiştirdi,” diyor Priya Shah.', `${QUOTE} diyor ${NAME}.`],+ ['This works for every team, says Tom Becker.', `This works for every team, says ${NAME}.`],+ ]) {+ expect(fix(line)).toBe(out);+ }+ // A known quote by a known person stays, a company after the name too.+ for (const line of [+ '“Forecasts are only as good as the deals in them,” says Ada Lovelace.',+ '„Forecasts are only as good as the deals in them“, sagt Ada Lovelace von Northwind.',+ // A company recommends too: its name isn't a person's.+ '“Forecasts are only as good as the deals in them,” recommends Northwind Finance.',+ ]) {+ expect(fix(line)).toBe(line);+ }+ });++ it('lets no invented name through on a known first word: only a known start before a particle', () => {+ for (const [given, line] of [+ [+ '“Grace Hopper wrote the first compiler,” the book says.',+ 'Dr. Grace Kim recommends a medium grind.',+ ],+ ['Selin Kaya, Head of Finance', 'Dr. Selin Arslan recommends weekly forecasts.'],+ ['Sarah Chen (CEO)', 'Dr. Sarah Kim runs the clinic.'],+ ['Mark the date: it will rain in May or June.', 'Dr. Mark Wilson agrees.'],+ ['Mark the date: it will rain in May or June.', 'Prof. Will Carter agrees.'],+ ['Mark the date: it will rain in May or June.', 'Dr. May Chen agrees.'],+ ['Mark the date: it will rain in May or June.', 'Dr. June Park agrees.'],+ // A known last name alone, or a common word as one.+ ['Grace Hopper', 'Dr. Jane Hopper recommends it.'],+ ['Green tea', 'Dr. Sarah Green recommends it.'],+ ['Selin Kaya', 'Dr. Ayşe Kaya recommends it.'],+ ]) {+ expect(guardPeople(line, knownPeople(given)).text).toContain(NAME);+ }+ expect(guardPeople('Dr. Hopper recommends it.', knownPeople('Grace Hopper')).text).toBe(+ 'Dr. Hopper recommends it.',+ );+ // A German noun after a known name, a word the page has: still the known name.+ const german = 'Wir rösten selbst, weil Gründerin Anna Schmidt Kaffee liebt.';+ expect(guardPeople(german, knownPeople('Anna Schmidt, Gründerin. Kaffee')).text).toBe(german);+ expect(guard('CEO Ada Lovelace Brown agrees.')).toBe(`CEO ${NAME} agrees.`);+ // A known name, a company after its particle: still known.+ expect(guard('CEO Ada Lovelace von Northwind agrees.')).toBe(+ 'CEO Ada Lovelace von Northwind agrees.',+ );+ });++ it('reads a Turkish role’s name before a comma or "da" when the sentence ends saying so', () => {+ expect(+ fix('Fintrail genel müdürü Ayşe Demir, bunun her ekip için işe yaradığını söyledi.'),+ ).toBe(`Fintrail genel müdürü ${NAME}, bunun her ekip için işe yaradığını söyledi.`);+ expect(fix('Fintrail genel müdürü Ayşe Demir, bunun önemli olduğunu ifade etti.')).toBe(+ `Fintrail genel müdürü ${NAME}, bunun önemli olduğunu ifade etti.`,+ );+ expect(+ fix('Fintrail genel müdürü Tom Becker da alıcıların çoğunun aynı fikirde olduğunu ekliyor.'),+ ).toBe(`Fintrail genel müdürü ${NAME} da alıcıların çoğunun aynı fikirde olduğunu ekliyor.`);+ // A verb said only of people needs no reported speech; one that also+ // means added needs it, plural and with "da" or "sözlerine" too.+ for (const [line, out] of [+ [+ 'Fintrail genel müdürü Ayşe Demir, “Bu her ekip için işe yarıyor” dedi.',+ `Fintrail genel müdürü ${NAME}, ${QUOTE} dedi.`,+ ],+ [+ 'Fintrail genel müdürü Ayşe Demir, ürünün çok hızlı olduğunu da ekledi.',+ `Fintrail genel müdürü ${NAME}, ürünün çok hızlı olduğunu da ekledi.`,+ ],+ [+ 'Fintrail genel müdürü Ayşe Demir, ekiplerin çok memnun olduklarını söyledi.',+ `Fintrail genel müdürü ${NAME}, ekiplerin çok memnun olduklarını söyledi.`,+ ],+ [+ 'Fintrail genel müdürü Ayşe Demir, bunun doğru olduğunu sözlerine ekledi.',+ `Fintrail genel müdürü ${NAME}, bunun doğru olduğunu sözlerine ekledi.`,+ ],+ [+ 'Kurucusu Ayşe Demir de bu yöntemi her ekibe anlatıyor.',+ `Kurucusu ${NAME} de bu yöntemi her ekibe anlatıyor.`,+ ],+ ]) {+ expect(fix(line)).toBe(out);+ }+ // A list of tools after a trade isn't a person, nor one "de dahil", nor+ // a name the sentence says nothing of.+ for (const line of [+ 'Bir SEO uzmanı Google Analytics, Search Console ve Ahrefs kullanır.',+ 'Bir e-ticaret uzmanı Shopify Plus, WooCommerce ve Magento arasındaki farkları açıklıyor.',+ 'Bir SEO uzmanı Google Analytics, Ahrefs ve Semrush verilerini birlikte okuduğunu söylüyor.',+ 'Pazarlama müdürü Google Analytics de dahil birçok aracı kullandığını söyledi.',+ 'Pazarlama müdürü Ayşe Yılmaz, kampanyayı yönetti.',+ // "ekledi", "kaydetti", "aktardı" mean added, saved, transferred too:+ // read only after reported speech ("…olduğunu ekliyor").+ 'Bir e-ticaret uzmanı Shopify Plus da kullanarak ürünleri mağazaya ekledi.',+ 'Bir veri uzmanı Google Sheets da kullanıp tabloyu kaydetti.',+ 'Proje yöneticisi Microsoft Teams da açık tutuyor ve dosyaları ekibe aktardı.',+ 'Hesap yöneticisi Google Ads da yeni kampanyayı ekledi.',+ ]) {+ expect(fix(line)).toBe(line);+ }+ });++ it('reads guillemets around a quote, never the words between two quotes, French spaced ones too', () => {+ for (const line of [+ '«Hızlı» ve «güvenilir» kelimeleri her yerde kullanılıyor ama müşteriler hep «sade» diyor.',+ 'Die Wörter »schnell« und »sicher« stehen überall, aber Kunden sagen »einfach«.',+ ]) {+ expect(fix(line)).toBe(line);+ }+ // French spaces its « … », with a normal, no-break or narrow no-break space.+ for (const line of [+ 'Les clients disent « rapide » mais ils veulent surtout un service « fiable » au quotidien.',+ 'On parle de « recherche générative » depuis deux ans, et le terme « AEO » se répand vite.',+ 'Le format « question-réponse » aide beaucoup les moteurs, surtout avec un bloc « FAQ » clair.',+ 'Les clients disent «\u00a0rapide\u00a0» mais ils veulent surtout un service «\u00a0fiable\u00a0» au quotidien.',+ 'Les clients disent «\u202frapide\u202f» mais ils veulent surtout un service «\u202ffiable\u202f» au quotidien.',+ ]) {+ expect(fix(line)).toBe(line);+ }+ expect(fix('Sie sagte: »Wir haben die Kosten in einem Jahr halbiert.«')).toBe(+ `Sie sagte: ${QUOTE}`,+ );+ // A quote they do hold is still read, a guillemet before it on the line too.+ expect(fix('«Haftalık tahmin her şeyi değiştirdi», diyor Priya Shah.')).toBe(+ `${QUOTE}, diyor ${NAME}.`,+ );+ expect(fix('Ürünler » Analiz: «Haftalık tahmin her şeyi değiştirdi», diyor Priya Shah.')).toBe(+ `Ürünler » Analiz: ${QUOTE}, diyor ${NAME}.`,+ );+ expect(+ fix('«Schnell» und »Das hat unseren Umsatz wirklich verdoppelt«, sagt Anna Weber.'),+ ).toBe(`«Schnell» und ${QUOTE}, sagt ${NAME}.`);+ });++ it('takes a product that "says" for a person: the trade made for "…, says Tom Becker."', () => {+ expect(fix('Broad match finds new queries, says Google Ads.')).toBe(+ `Broad match finds new queries, says ${NAME}.`,+ );+ });++ it("reads a customer's story as a person's, and no customer in a product's name", () => {+ for (const [line, out] of [+ [+ 'Last spring our customer Jana Novak finished in a week.',+ `Last spring our customer ${NAME} finished in a week.`,+ ],+ ['One client Jana Novak called it simple.', `One client ${NAME} called it simple.`],+ ['Müşterimiz Jana Novak işi erken bitirdi.', `Müşterimiz ${NAME} işi erken bitirdi.`],+ ['Unsere Kundin Jana Novak war früher fertig.', `Unsere Kundin ${NAME} war früher fertig.`],+ ['Ein Kunde Jonas Weber war früher fertig.', `Ein Kunde ${NAME} war früher fertig.`],+ ]) {+ expect(fix(line)).toBe(out);+ }+ for (const line of [+ 'A Customer Data Platform joins the data.',+ 'HubSpot, a customer relationship management platform, tracks deals.',+ 'Microsoft Outlook, an email client, syncs mail.',+ 'The email client Microsoft Outlook syncs mail.',+ 'Track customer Lifetime Value monthly.',+ 'Salesforce Service Cloud, müşteri desteği için bir platformdur.',+ 'Müşteri Deneyimi Yönetimi önemlidir.',+ ]) {+ expect(fix(line)).toBe(line);+ }+ });++ it('reads German compound posts and "erzählte", and no ladder, conductor, product or newspaper', () => {+ for (const [line, out] of [+ [+ 'Marco Rossi, Finanzchef bei Fintrail, sagte, dass es funktioniert.',+ `${NAME}, Finanzchef bei Fintrail, sagte, dass es funktioniert.`,+ ],+ [+ 'Laut Vertriebsleiterin Anna Schmidt läuft es gut.',+ `Laut Vertriebsleiterin ${NAME} läuft es gut.`,+ ],+ ['Anna Schmidt erzählte uns davon.', `${NAME} erzählte uns davon.`],+ ]) {+ expect(fix(line)).toBe(out);+ }+ for (const line of [+ 'Die Leiter Hailo ProfiStep steht im Keller.',+ 'Die Teleskopleiter Hailo ProfiStep steht im Keller.',+ 'Der Halbleiter Infineon Technologies liefert Chips.',+ 'Der Schutzleiter Phoenix Contact ist grün-gelb.',+ 'Bosch Rexroth, ein Hersteller für Halbleiter, wächst.',+ 'Use Chef Infra Server to manage your fleet.',+ 'Wie Der Spiegel berichtete, wächst der Markt.',+ 'Die Süddeutsche Zeitung berichtete über das Tool.',+ 'Microsoft Deutschland erklärte, dass Copilot kommt.',+ ]) {+ expect(fix(line)).toBe(line);+ }+ });++ it('reads the new cues quickly on long text', () => {+ const long =+ `Fintrail genel müdürü Ayşe Demir, ${'çok uzun bir açıklama yapıyor '.repeat(8)}`.repeat(400);+ const started = Date.now();+ guardPeople(`${long} “${'word '.repeat(50)}” says ${'Aa Bb, '.repeat(200)}`, known);+ expect(Date.now() - started).toBeLessThan(2000);+ });+});+describe('guardPeopleInHtml', () => {it('reads the words only, and makes an unknown blockquote one placeholder', () => {const { html, count } = guardPeopleInHtml(diff --git a/server/src/lib/content-sources.js b/server/src/lib/content-sources.jsindex ecac1dce2..9dbf57b72 100644--- a/server/src/lib/content-sources.js+++ b/server/src/lib/content-sources.js@@ -118,10 +118,14 @@ const CURRENCY = /[$€£¥₺]/;const UNIT_AFTER =/^\s?(?:%|×|[xX]\b|[kKMB]\b|bn\b|mn\b|[Mm]illions?\b|[Bb]illions?\b|[Tt]housands?\b|[Tt]rillions?\b)/;-/** A claim's unit, one name for each way it is written: "40%", "3x", "2M", "$40". */+/** A percent written before its number, as Turkish does: "%40", "yüzde 40". */+const PERCENT_BEFORE = /(?:%\s?|(?<!\p{L})yüzde\s)$/iu;++/** A claim's unit, one name for each way it is written: "40%", "%40", "3x", "2M", "$40". */function unitOf(text, start, end) {const before = text[start - 1] ?? '';if (CURRENCY.test(before)) return before;+ if (PERCENT_BEFORE.test(text.slice(Math.max(0, start - 6), start))) return '%';const unit = text.slice(end).match(UNIT_AFTER)?.[0].trim().toLowerCase();if (!unit) return '';if (unit === '%') return '%';@@ -323,10 +327,12 @@ export function placeholderNumbers(text, known) {const added = addedNumbers(text, known);let out = String(text ?? '');for (const n of added.reverse()) {- // The sign and unit go with it: "$40" and "40%" become the placeholder.+ // The sign and unit go with it: "$40", "40%" and "%40" become the placeholder.let start = n.start;let end = n.end;if (CURRENCY.test(out[start - 1] ?? '')) start -= 1;+ const percent = out.slice(Math.max(0, start - 2), start).match(/%\s?$/);+ if (percent) start -= percent[0].length;const unit = out.slice(end).match(UNIT_AFTER);if (unit) end += unit[0].length;out = `${out.slice(0, start)}${NUMBER_PLACEHOLDER}${out.slice(end)}`;diff --git a/server/src/lib/content-writer.js b/server/src/lib/content-writer.jsindex b811c1b4e..65a059376 100644--- a/server/src/lib/content-writer.js+++ b/server/src/lib/content-writer.js@@ -123,7 +123,7 @@ export const pageSchema = z.object({faq: z.array(z.object({ question: z.string(), answer: z.string() })).max(8),});-const WRITE_SYSTEM_PROMPT = `You are the Content Agent for Answer Engine Optimization (AEO). You write one web page so AI assistants (ChatGPT, Perplexity, Gemini, Copilot, Google's AI Overviews) understand it, trust it and cite it. A person reviews it before it is published.+export const WRITE_SYSTEM_PROMPT = `You are the Content Agent for Answer Engine Optimization (AEO). You write one web page so AI assistants (ChatGPT, Perplexity, Gemini, Copilot, Google's AI Overviews) understand it, trust it and cite it. A person reviews it before it is published.How to write:- Answer first: the opening paragraphs answer the page's main question directly.diff --git a/server/src/lib/no-fabrication.test.js b/server/src/lib/no-fabrication.test.jsnew file mode 100644index 000000000..778973c14--- /dev/null+++ b/server/src/lib/no-fabrication.test.js@@ -0,0 +1,157 @@+import { existsSync, readFileSync } from 'fs';+import { describe, expect, it, vi } from 'vitest';+import { CASES } from './__fixtures__/no-fabrication/cases.js';++/**+ * No invented people, quotes or numbers (Content Editor Quality spec §9):+ * the Content Agent's answers on every draft in cases.js go through the guards+ * as its runs put them (keepSuggestions for its suggestions, sourcedPage+ * for a page it writes whole), with what it was given: the page, the+ * brand, the authors, the Sources tab's data and Brief › Your input. None+ * of what it made up may survive, and none of what it was given may be+ * taken out. Answers recorded from the model (eval-no-fabrication.js) are+ * read the same way: the guards must find nothing left to take out of what+ * they let through (they are stable), and what they let through that the+ * agent wasn't given must be what a person has looked over (toReview, kept+ * in recorded.json as `reviewed`): the guards' cues can't vouch for+ * themselves.+ */++vi.mock('../config/supabase.js', () => ({ default: {} }));++const { draftPassages, keepSuggestions } = await import('./content-agent.js');+const { peopleEdits } = await import('./content-people.js');+const { addedNumbers } = await import('./content-sources.js');+const { guardedFix, guardedPage, plain, resolve, sourcing, toReview } =+ await import('./__fixtures__/no-fabrication/guarded.js');++/** What the agent's suggestions leave on the page, and what a page it writes says. */+function outputs(c) {+ return { fix: guardedFix(c, c.fix).join('\n'), page: guardedPage(c, c.page) };+}++describe('the no-fabrication test set', () => {+ it('holds 50 drafts at least, of each site type and language', () => {+ expect(CASES.length).toBeGreaterThanOrEqual(50);+ expect(new Set(CASES.map((c) => c.siteType))).toEqual(new Set(['b2b', 'b2c', 'ecommerce']));+ expect(new Set(CASES.map((c) => c.language))).toEqual(new Set(['en', 'tr', 'de']));+ expect(new Set(CASES.map((c) => c.id)).size).toBe(CASES.length);+ });++ it('drops a suggestion only for a figure it made up: the rest reach the guards', () => {+ const dropped = [];+ for (const c of CASES) {+ const { numbered, known, people } = sourcing(c);+ for (const s of c.fix.map(resolve(numbered))) {+ const kept = keepSuggestions(+ [s],+ draftPassages(c.html),+ new Set(),+ new Set(['dated']),+ null,+ {+ numbered,+ known,+ people,+ },+ );+ const madeUpFigure = c.invented.some((item) => /\d/.test(item) && s.after.includes(item));+ if (!kept.length && !madeUpFigure) dropped.push(`${c.id}: ${s.kind} on ${s.passage}`);+ }+ }+ expect(dropped).toEqual([]);+ });++ it('lets nothing the agent made up through, and takes out nothing it was given', () => {+ const survivors = [];+ const lost = [];+ for (const c of CASES) {+ const out = outputs(c);+ for (const [path, text] of Object.entries(out)) {+ for (const item of c.invented) {+ if (plain(text).includes(plain(item))) survivors.push(`${c.id} (${path}): ${item}`);+ }+ for (const item of c.kept[path]) {+ if (!plain(text).includes(plain(item))) lost.push(`${c.id} (${path}): ${item}`);+ }+ }+ }+ expect(survivors).toEqual([]);+ expect(lost).toEqual([]);+ });++ it('lists for review, without the cues, the names and quotes it wasn’t given', () => {+ const [c] = CASES.filter((x) => x.id === 'b2b-en-given_input');+ // What the set's own answers keep: names it was given only.+ const listed = CASES.flatMap((x) =>+ [...guardedFix(x, x.fix), guardedPage(x, x.page)].flatMap((text) =>+ toReview(x, text).map((span) => `${x.id}: ${span}`),+ ),+ );+ expect([...new Set(listed)]).toEqual([+ 'b2b-tr-given_input: Northwind Ada Lovelace',+ 'b2b-de-invented_story: Unsere Kundin',+ 'b2c-tr-given_input: Bean Lab Grace',+ 'b2c-tr-given_input: Baş Kavurmacı Mert',+ 'b2c-de-invented_story: Unsere Kundin',+ 'ecommerce-tr-given_input: Velo Studio Margaret',+ 'ecommerce-tr-given_input: Atölye Sorumlusu Lena',+ 'ecommerce-de-invented_story: Unsere Kundin',+ ]);+ // A quote with an apostrophe in it, a name after a heading, a name in a+ // given word ("parking" isn't "Park").+ expect(toReview(c, '“We don’t invent numbers or people, ever,” says the team.')).toEqual([+ 'We don’t invent numbers or people, ever',+ ]);+ expect(+ toReview(c, guardedPage(c, { ...c.page, intro: ['Melanie Ross recommends it.'] })),+ ).toContain('Melanie Ross');+ expect(toReview(c, 'We asked Ed Park.')).toEqual(['Ed Park']);+ // What it wasn't given, unguarded: every made-up name and quote is listed.+ for (const x of CASES) {+ const spans = toReview(x, [...x.fix.map((s) => s.after), ...x.page.intro].join('\n')).join(+ '\n',+ );+ for (const item of x.invented.filter((i) => !/\d/.test(i))) {+ expect(spans, `${x.id}: ${item}`).toContain(item.split(/\s+/).slice(0, 3).join(' '));+ }+ }+ });++ // The model's own answers, once recorded (not in CI: it needs a model's key).+ const recordedPath = new URL('./__fixtures__/no-fabrication/recorded.json', import.meta.url);+ it.skipIf(!existsSync(recordedPath))(+ "leaves in the model's own answers no figure, and no name or quote it wasn't given that a person hasn't looked over",+ () => {+ const recorded = JSON.parse(readFileSync(recordedPath, 'utf8'));+ const survivors = [];+ let read = 0;+ for (const c of CASES) {+ const answer = recorded[c.id];+ if (!answer) continue;+ read += 1;+ const { people, knownInPage } = sourcing(c);+ const fix = guardedFix(c, answer.fix ?? [], new Set(['dated', 'expert', 'example']));+ const page = answer.page ? guardedPage(c, answer.page) : '';+ const reviewed = new Set(recorded.reviewed?.[c.id] ?? []);+ for (const text of [...fix, page]) {+ // What the guards would still take out of what they let through:+ // nothing, or they aren't stable.+ for (const n of addedNumbers(text, knownInPage)) {+ survivors.push(`${c.id}: figure ${text.slice(n.start, n.end)}`);+ }+ for (const e of peopleEdits(text, people)) {+ survivors.push(`${c.id}: ${text.slice(e.start, e.end)}`);+ }+ // Names and quotes it wasn't given, read without the guards' cues:+ // each one a person has looked over.+ for (const span of toReview(c, text)) {+ if (!reviewed.has(span)) survivors.push(`${c.id}: not reviewed: ${span}`);+ }+ }+ }+ expect(read).toBeGreaterThan(0);+ expect(survivors).toEqual([]);+ },+ );+});diff --git a/server/src/scripts/eval-no-fabrication.js b/server/src/scripts/eval-no-fabrication.jsnew file mode 100644index 000000000..434fda0d1--- /dev/null+++ b/server/src/scripts/eval-no-fabrication.js@@ -0,0 +1,139 @@+/**+ * Record the Content Agent's own answers on the no-fabrication test set+ * (src/lib/__fixtures__/no-fabrication/cases.js, Quality spec §9), for+ * no-fabrication.test.js to read: every figure that survives the guards+ * must be in what the agent was given, and every name or quote it wasn't+ * given that they let through must be one a person has looked over. The+ * script prints those (toReview: read without the guards' cues); copy the+ * ones checked into recorded.json's `reviewed`, by draft.+ *+ * Run from server/, with .env holding the content model's provider key (the+ * model is IMPROVE_SUGGESTION_MODEL, else DEFAULT_SUGGESTION_MODEL, else+ * google/gemini-3-flash-preview) and the Supabase keys the agent's modules+ * load with (nothing is read or written there):+ * node src/scripts/eval-no-fabrication.js [count]+ * count defaults to every draft: two calls each, the suggestions Fix+ * open rules asks for and a page written whole, as Start asks for it.+ *+ * Writes src/lib/__fixtures__/no-fabrication/recorded.json, only when at+ * least one draft was answered. Reads and writes no database, never prints a+ * key, and isn't run in CI: the answers it keeps+ * are the model's of the day, for a person to re-record when the prompts or+ * the model change. A few cents at today's prices.+ */+import 'dotenv/config';+import { writeFileSync } from 'fs';+import { generateObject } from 'ai';+import { CASES } from '../lib/__fixtures__/no-fabrication/cases.js';+import { guardedFix, guardedPage, toReview } from '../lib/__fixtures__/no-fabrication/guarded.js';+import { resolveModel } from '../lib/ai-provider.js';+import {+ SYSTEM_PROMPT,+ buildFixPrompt,+ draftPassages,+ suggestionsSchema,+} from '../lib/content-agent.js';+import { contentBriefModelId } from '../lib/content-brief.js';+import { inputSources } from '../lib/content-inputs.js';+import { numberSources, sourcesPrompt } from '../lib/content-sources.js';+import {+ WRITE_SYSTEM_PROMPT,+ briefText,+ buildWritePrompt,+ pageSchema,+} from '../lib/content-writer.js';++const count = Math.max(+ 1,+ Math.min(CASES.length, Number.parseInt(process.argv[2] ?? `${CASES.length}`, 10) || CASES.length),+);++/** The open rules: the ones that tempt a model to make things up. */+const RULES = [+ { key: 'dated', name: 'A dated data point', detail: 'missing' },+ { key: 'expert', name: 'A quote from an expert', detail: 'missing' },+ { key: 'example', name: 'A real customer example', detail: 'missing' },+];++const modelId = contentBriefModelId();+const model = resolveModel(modelId);+const recorded = {};+const started = Date.now();++for (const c of CASES.slice(0, count)) {+ const numbered = numberSources({+ prompts: c.prompts,+ fanouts: c.fanouts,+ inputs: inputSources(c.inputs, c.brand.name),+ });+ const draft = { title: c.title, page_type: 'blog_post', body_html: c.html };+ const brand = { ...c.brand, language: c.language, site_type: c.siteType };+ const input = {+ rules: RULES,+ textRules: [],+ locked: [],+ sources: {+ promptIds: [],+ fanouts: c.fanouts,+ pages: [],+ queries: [],+ headings: [],+ length: 2500,+ },+ };+ try {+ const [fix, page] = await Promise.all([+ generateObject({+ model,+ schema: suggestionsSchema,+ system: SYSTEM_PROMPT,+ prompt: buildFixPrompt({+ brand,+ draft,+ passages: draftPassages(c.html),+ lockedKeys: new Set(),+ input,+ sources: sourcesPrompt(numbered),+ }),+ }),+ generateObject({+ model,+ schema: pageSchema,+ system: WRITE_SYSTEM_PROMPT,+ prompt: buildWritePrompt({ brand, draft, numbered, brief: briefText(null, null), input }),+ }),+ ]);+ recorded[c.id] = { fix: fix.object.suggestions, page: page.object };+ console.log(`${c.id}: ${fix.object.suggestions.length} suggestions and a page`);+ } catch (err) {+ console.error(`${c.id}: failed (${err?.message ?? err})`);+ }+}++if (!Object.keys(recorded).length) {+ console.error('No draft was answered: recorded.json is left as it was.');+ process.exit(1);+}+writeFileSync(+ new URL('../lib/__fixtures__/no-fabrication/recorded.json', import.meta.url),+ `${JSON.stringify({ model: modelId, recordedAt: new Date().toISOString(), reviewed: {}, ...recorded }, null, 2)}\n`,+);+// Then what the guards let through that the agent wasn't given, for a+// person to look over: the answers are kept first, whatever happens here.+const review = {};+for (const c of CASES) {+ const answer = recorded[c.id];+ if (!answer) continue;+ const texts = [+ ...guardedFix(c, answer.fix, new Set(RULES.map((r) => r.key))),+ guardedPage(c, answer.page),+ ];+ const spans = [...new Set(texts.flatMap((text) => toReview(c, text)))];+ if (spans.length) review[c.id] = spans;+}+for (const [id, spans] of Object.entries(review)) {+ console.log(`To review, ${id}:\n${spans.map((span) => ` ${span}`).join('\n')}`);+}+console.log(+ `Recorded ${Object.keys(recorded).length} of ${count} drafts with ${modelId} in ${((Date.now() - started) / 1000).toFixed(0)} s.`,+);