Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(content): count a Hindi word with its vowel signs as one in the rules, as Readability does (#1411 follow-up) (#1444)
Cihanf87687fef621
diff --git a/web/src/lib/content/readability.ts b/web/src/lib/content/readability.tsindex f0d2c24c5..4b02db9d2 100644--- a/web/src/lib/content/readability.ts+++ b/web/src/lib/content/readability.ts@@ -80,12 +80,13 @@ export interface ReadBlock {}/**- * Words as the rules count them (rules.ts countWords), a vowel sign or other- * mark kept inside its word: Hindi's are marks, not letters, and would break- * each word in two or three.+ * Words as Readability and the rules count them: letters or digits, with+ * inner apostrophes and hyphens, a vowel sign or other mark kept inside its+ * word (Hindi's are marks, not letters, and would break each word in two or+ * three).*/const WORD = /[\p{L}\p{N}][\p{L}\p{M}\p{N}'’-]*/gu;-const countWords = (text: string) => text.match(WORD)?.length ?? 0;+export const countWords = (text: string) => text.match(WORD)?.length ?? 0;/*** Text joined as a reader sees it: a space between paragraphs and at a linediff --git a/web/src/lib/content/rules.test.ts b/web/src/lib/content/rules.test.tsindex d0bb72d4f..fcf56f46e 100644--- a/web/src/lib/content/rules.test.ts+++ b/web/src/lib/content/rules.test.ts@@ -143,6 +143,12 @@ describe('rule sets', () => {expect(ruleLength(huge, 'category_page')).toBe(50_000);});+ it('counts a word with its vowel signs as one, as Readability does (Hindi)', () => {+ // Hindi's vowel signs are marks, not letters: the words split at them before.+ expect(page('<p>यह पहला वाक्य है। यह दूसरा है।</p>').blocks[0].words).toBe(7);+ expect(page("<p>Don't split well-known words.</p>").blocks[0].words).toBe(4);+ });+it('says a length in characters as about so many words (v1.1 §3.2)', () => {// To the nearest ten from 100 words on, as the Brief shows it.expect(wordsForLength(4000)).toBe(670);diff --git a/web/src/lib/content/rules.ts b/web/src/lib/content/rules.tsindex ea74e9043..a7cb2513f 100644--- a/web/src/lib/content/rules.ts+++ b/web/src/lib/content/rules.ts@@ -4,6 +4,7 @@ import { classifyDomain, normalizeDomain } from '@/lib/citations/classify';import { PAGE_TYPES, type DraftMethod, type PageType } from '@/lib/content/drafts';import { lengthTarget } from '@/lib/content/editor';import { decodeHtmlBlock } from '@/lib/content/html-allowlist';+import { countWords } from '@/lib/content/readability';import { wordsOf } from '@/lib/content/sources';/**@@ -572,11 +573,6 @@ export function readPage(html: string, characters: number): PageRead {};}-/** Words as the rules count them: letters or digits, with inner apostrophes and hyphens. */-export function countWords(text: string): number {- return text.match(/[\p{L}\p{N}][\p{L}\p{N}'’-]*/gu)?.length ?? 0;-}-const isHeading = (b: PageBlock) => b.tag === 'h1' || b.tag === 'h2' || b.tag === 'h3';const isText = (b: PageBlock) => b.tag === 'p' || b.tag === 'li';const isQuestion = (b: PageBlock) => b.text.trim().endsWith('?');