Free, open source blog writer for Claude Code. Researches current sources, writes answer-first posts for Google and AI answers, has an independent reviewer score each one out of 100 and ships only after five delivery gates. 32 skills, 30 /blog commands, 5 agents. Active development at AI-Marketing-Hub/claude-blog; public releases ship here.
Python agent-skillsaiai-citationsai-contentai-marketingai-marketing-hubblogblog-writingclaude-codeclaude-code-skillclaude-pluginclaude-skillcontent-creationcontent-optimizationcontent-strategyeeatgeomultilingualopen-sourceseo
#!/usr/bin/env python3
"""
Blog Quality Analyzer - 5-Category, 100-Point Scoring System
Analyzes blog post files for content quality, SEO optimization, E-E-A-T signals,
technical elements, and AI citation readiness. Returns structured JSON, markdown
reports, or compact tables.
Usage:
python3 analyze_blog.py <file> # Default JSON output
python3 analyze_blog.py <file> --format markdown # Markdown report
python3 analyze_blog.py <file> --format table # Compact table
python3 analyze_blog.py <directory> --batch --sort score # Batch with sorting
python3 analyze_blog.py <file> --category seo # Single category detail
python3 analyze_blog.py <file> --fix # Output specific fixes
Scoring:
Content Quality 30 pts Coverage, readability, originality, structure, utility, grammar
SEO Optimization 25 pts Title, headings, keywords, linking, meta, URL
E-E-A-T Signals 15 pts Author, citations, trust, evidence basis
Technical Elements 15 pts Schema, images, structured data, speed, mobile, social
AI Citation Readiness 15 pts Evidence, purpose, entities, utility, crawler access
Bands:
90-100 Exceptional
80-89 Strong
70-79 Acceptable
60-69 Below Standard
<60 Rewrite
Optional dependencies (graceful degradation):
pip install textstat beautifulsoup4
"""
import argparse
import errno
import json
import os
import re
import stat
import sys
import tempfile
import urllib.parse
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
def _project_version() -> str:
"""Read the package version from pyproject.toml."""
try:
text = (Path(__file__).resolve().parent.parent / "pyproject.toml").read_text(encoding="utf-8")
except OSError:
return "0.0.0"
match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE)
return match.group(1) if match else "0.0.0"
# ---------------------------------------------------------------------------
# Optional dependency detection
# ---------------------------------------------------------------------------
try:
import textstat
HAS_TEXTSTAT = True
except ImportError:
HAS_TEXTSTAT = False
try:
from bs4 import BeautifulSoup
HAS_BS4 = True
except ImportError:
HAS_BS4 = False
def _print_dependency_notice() -> None:
"""Print missing-dependency notice to stderr so JSON output stays clean."""
missing: list[str] = []
if not HAS_TEXTSTAT:
missing.append('textstat')
if not HAS_BS4:
missing.append('beautifulsoup4')
if missing:
print(
f"Note: Optional dependencies not found: {', '.join(missing)}. "
f"Install with: pip install {' '.join(missing)}",
file=sys.stderr,
)
# ---------------------------------------------------------------------------
# Editorial style diagnostic phrases
# ---------------------------------------------------------------------------
AI_PHRASES = [
"in today's digital landscape", "it's important to note", "in conclusion",
"dive into", "game-changer", "navigate the landscape", "revolutionize",
"leverage", "comprehensive guide", "in the ever-evolving", "seamlessly",
"cutting-edge", "harness the power", "at its core", "rich tapestry",
"empower", "state-of-the-art",
]
# Configurable editorial terms to flag for manual review.
AI_TRIGGER_WORDS = [
"delve", "tapestry", "multifaceted", "testament", "pivotal", "robust",
"cutting-edge", "furthermore", "indeed", "moreover", "utilize", "leverage",
"comprehensive", "landscape", "crucial", "foster", "illuminate", "underscore",
"embark", "endeavor", "facilitate", "paramount", "nuanced", "intricate",
"meticulous", "realm",
]
# Transition words/phrases for readability scoring
TRANSITION_WORDS = [
"however", "therefore", "furthermore", "moreover", "additionally",
"consequently", "nevertheless", "meanwhile", "similarly", "likewise",
"nonetheless", "accordingly", "subsequently", "hence", "thus",
"in contrast", "on the other hand", "for example", "for instance",
"in addition", "as a result", "in other words", "that said",
"in particular", "specifically", "alternatively", "conversely",
"in fact", "notably", "importantly", "significantly",
]
# ---------------------------------------------------------------------------
# Content type word-count benchmarks
# ---------------------------------------------------------------------------
CONTENT_TYPE_BENCHMARKS: dict[str, tuple[int, int]] = {
'guide': (2500, 5000),
'how-to': (1500, 3000),
'listicle': (1200, 2500),
'opinion': (800, 1500),
'case-study': (1500, 3000),
'news': (600, 1200),
'review': (1000, 2000),
'default': (1200, 3000),
}
# ---------------------------------------------------------------------------
# Source tier classification
# ---------------------------------------------------------------------------
MAX_INPUT_BYTES = 10 * 1024 * 1024
TIER1_DOMAINS = [
'nature.com', 'science.org', 'gov', 'edu', 'who.int', 'nih.gov',
'cdc.gov', 'bls.gov', 'census.gov', 'europa.eu', 'un.org',
'ieee.org', 'acm.org', 'arxiv.org', 'pubmed.ncbi',
]
TIER2_DOMAINS = [
'reuters.com', 'apnews.com', 'bbc.com', 'nytimes.com',
'washingtonpost.com', 'economist.com', 'forbes.com', 'hbr.org',
'mckinsey.com', 'gartner.com', 'statista.com', 'pew', 'gallup.com',
]
LANGUAGE_PROFILES: dict[str, dict[str, Any]] = {
'en': {
'summary_labels': (
r'TL;?DR', r'key takeaway', r'the bottom line',
r'what you.ll learn', r'at a glance', r'in brief',
),
'about_patterns': (
r'\babout\s+(?:us|the author|me)\b', r'/about(?:[/?#]|$)',
),
'contact_patterns': (r'\bcontact\b', r'/contact(?:[/?#]|$)'),
'first_person_patterns': (
r'\bI\s+(?:found|discovered|tested|built|created|noticed|learned|experienced)\b',
r'\b(?:we|our team)\s+(?:tested|built|ran|analyzed|measured|conducted|found|discovered)\b',
r'\bin (?:my|our) experience\b',
r'\bfrom (?:my|our) (?:testing|research|analysis|work)\b',
),
'methodology_patterns': (
r'\b(?:methodology|sample size|test setup|testing method|research method)\b',
r'\b(?:we|I|our team)\s+(?:tested|measured|analyzed|conducted)\b[^.\n]{0,180}'
r'(?:\d|https?://|\[[^\]\n]{1,300}\]\(https?://)',
),
'readability_model': 'flesch',
},
'id': {
# Indonesian is selected only when declared in frontmatter because it
# has no distinctive markers reliable enough for automatic detection.
'summary_labels': (
r'jawaban singkat', r'ringkasan', r'intinya', r'singkatnya', r'TL;?DR',
),
'about_patterns': (
r'/tentang(?:-kami)?(?:[/?#]|$)', r'\btentang kami\b',
r'\bprofil (?:perusahaan|kami)\b',
),
'contact_patterns': (
r'/kontak(?:[/?#]|$)', r'/hubungi(?:-kami)?(?:[/?#]|$)',
r'\bhubungi kami\b', r'\bkontak kami\b',
),
'first_person_patterns': (
r'\b(?:kami|tim kami)\s+(?:menguji|mengaudit|membedah|mengukur|menemukan|'
r'menangani|mencatat|menganalisis|membandingkan|mengelola)\b',
r'\b(?:dari|berdasarkan|menurut)\s+pengalaman kami\b',
r'\bkasus yang (?:paling )?kami (?:ingat|tangani|temui)\b',
r'\bdi akun (?:klien )?yang kami (?:audit|kelola|pegang)\b',
r'\bdata internal(?: kami)?\b',
),
'methodology_patterns': (
r'\b(?:metode|metodologi|sampel|ukuran sampel|data internal|'
r'dari \d+ (?:akun|klien|kampanye))\b',
r'\b(?:kami|tim kami)\s+(?:menguji|mengukur|menganalisis|mengaudit|'
r'membedah|membandingkan)\b[^.\n]{0,180}'
r'(?:\d|https?://|\[[^\]\n]{1,300}\]\(https?://)',
),
# Keep the existing fallback until a validated Indonesian readability
# formula is available in this analyzer.
'readability_model': 'flesch',
},
'tr': {
'summary_labels': (r'özet', r'özetle', r'kısaca'),
'about_patterns': (
r'/biz-kimiz(?:[/?#]|$)', r'/hakk[ıi]m[ıi]zda(?:[/?#]|$)',
r'\bbiz kimiz\b', r'\bhakk[ıi]m[ıi]zda\b',
),
'contact_patterns': (
r'/ilet[iİ]ş[iİ]m(?:[/?#]|$)', r'\bilet[iİ]ş[iİ]m\b',
),
'first_person_patterns': (
r'\b(?:biz|ekibimiz)\s+(?:test ettik|oluşturduk|inceledik|ölçtük|analiz ettik|derledik|bulduk)\b',
r'\b(?:kendi\s+)?(?:kayıtlarımızdan|verilerimizden|analizimizden)\s+'
r'(?:derlediğimiz|çıkardığımız|ölçtüğümüz)\b',
r'\bportfolyomuzun\b[^.\n]{0,120}\bsayımından çıkarıldı\b',
r'\b(?:deneyimimize|tecrübemize) göre\b',
),
'methodology_patterns': (
r'\b(?:yöntem|metodoloji|örneklem|örneklem büyüklüğü|test düzeni|araştırma yöntemi|sayım|kayıt)\b',
r'\b(?:biz|ekibimiz)\s+(?:test ettik|ölçtük|analiz ettik)\b[^.\n]{0,180}'
r'(?:\d|https?://|\[[^\]\n]{1,300}\]\(https?://)',
),
'readability_model': 'atesman',
},
}
def _read_safely(path: Path, max_bytes: int = MAX_INPUT_BYTES) -> str:
"""Read a regular non-symlink file with a size cap."""
flags = os.O_RDONLY
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
elif path.is_symlink():
raise ValueError(f"refusing to follow symlink: {path}")
try:
fd = os.open(str(path), flags)
except OSError as exc:
if exc.errno == errno.ELOOP:
raise ValueError(f"refusing to follow symlink: {path}") from exc
raise
try:
st = os.fstat(fd)
if not stat.S_ISREG(st.st_mode):
raise ValueError(f"not a regular file: {path}")
if st.st_size > max_bytes:
raise ValueError(f"input exceeds size cap ({st.st_size} > {max_bytes}): {path}")
data = os.read(fd, max_bytes + 1)
finally:
os.close(fd)
if len(data) > max_bytes:
raise ValueError(f"input exceeds size cap after read ({max_bytes}): {path}")
return data.decode("utf-8")
def _safe_write_text(path: str | Path, text: str) -> None:
"""Write output atomically and refuse symlink targets."""
out = Path(path)
if out.exists() and out.is_symlink():
raise ValueError(f"output path is a symlink: {out}")
if not out.parent.exists():
raise ValueError(f"output directory does not exist: {out.parent}")
fd, tmp = tempfile.mkstemp(dir=str(out.parent), prefix=f".{out.name}.", suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write(text)
os.replace(tmp, out)
except Exception:
try:
os.unlink(tmp)
except OSError:
pass
raise
def _hostname(url: str) -> str:
try:
parsed = urllib.parse.urlparse(url)
except ValueError:
return ""
return (parsed.hostname or "").lower().rstrip(".")
def _host_matches_domain(host: str, domain: str) -> bool:
domain = domain.lower().rstrip(".")
if domain in {"gov", "edu"}:
if host == domain or host.endswith(f".{domain}"):
return True
labels = host.split(".")
country_suffix = len(labels[-1]) == 2 and labels[-1].isalpha()
if country_suffix and len(labels) >= 2 and labels[-2] == domain:
return True
if domain == "edu" and country_suffix and len(labels) >= 2 and labels[-2] == "ac":
return True
return False
return host == domain or host.endswith(f".{domain}")
# ---------------------------------------------------------------------------
# Frontmatter extraction (kept from original)
# ---------------------------------------------------------------------------
def extract_frontmatter(content: str) -> dict[str, Any]:
"""Extract YAML frontmatter from markdown/MDX content."""
frontmatter: dict[str, Any] = {}
match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
if match:
fm_text = match.group(1)
for line in fm_text.split('\n'):
if ':' in line:
key, _, value = line.partition(':')
key = key.strip()
value = value.strip().strip('"').strip("'")
if value:
frontmatter[key] = value
return frontmatter
def strip_frontmatter(content: str) -> str:
"""Remove YAML frontmatter from content."""
return re.sub(r'^---\s*\n.*?\n---\s*\n', '', content, count=1, flags=re.DOTALL)
class _HTMLAnalysisParser(HTMLParser):
"""Extract page metadata and a Markdown-like reader-visible HTML view."""
_SKIPPED_TAGS = {'script', 'style', 'template', 'noscript', 'svg'}
_BLOCK_TAGS = {
'article', 'aside', 'div', 'footer', 'header', 'main', 'nav', 'p',
'section',
}
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.metadata: dict[str, str] = {}
self._metadata_priority: dict[str, int] = {}
self._parts: list[str] = []
self._in_head = False
self._in_title = False
self._title_parts: list[str] = []
self._skip_depth = 0
self._anchors: list[str] = []
self._lists: list[str] = []
self._table_depth = 0
self._table_first_row = False
self._table_row_cells = 0
self._json_ld_parts: list[str] | None = None
def _set_metadata(self, key: str, value: Any, priority: int) -> None:
text = str(value or '').strip()
if text and priority > self._metadata_priority.get(key, -1):
self.metadata[key] = text
self._metadata_priority[key] = priority
def _newline(self, count: int = 1) -> None:
if not self._parts:
return
trailing = len(self._parts[-1]) - len(self._parts[-1].rstrip('\n'))
if trailing < count:
self._parts.append('\n' * (count - trailing))
def _extract_json_ld_metadata(self, raw: str) -> None:
try:
data = json.loads(raw)
except (json.JSONDecodeError, TypeError, RecursionError):
return
def visit(value: Any) -> None:
if isinstance(value, dict):
# JSON-LD is untrusted: "@type" may be any JSON value.
schema_type = value.get('@type', '')
if isinstance(schema_type, str):
types = {schema_type}
elif isinstance(schema_type, list):
types = {item for item in schema_type if isinstance(item, str)}
else:
types = set()
priority = 2 if types & {'Article', 'BlogPosting', 'NewsArticle'} else 1
self._set_metadata('title', value.get('headline') or value.get('name'), priority)
self._set_metadata('description', value.get('description'), priority)
self._set_metadata('date', value.get('datePublished'), priority)
self._set_metadata('lastUpdated', value.get('dateModified'), priority)
self._set_metadata('language', value.get('inLanguage'), priority)
author = value.get('author')
if isinstance(author, dict):
self._set_metadata('author', author.get('name'), priority)
elif isinstance(author, list):
names = [str(item.get('name', '')).strip() for item in author if isinstance(item, dict)]
self._set_metadata('author', ', '.join(name for name in names if name), priority)
elif isinstance(author, str):
self._set_metadata('author', author, priority)
for child in value.values():
if isinstance(child, (dict, list)):
visit(child)
elif isinstance(value, list):
for item in value:
visit(item)
try:
visit(data)
except RecursionError:
return
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
tag = tag.lower()
values = {str(key).lower(): value or '' for key, value in attrs}
if tag == 'html':
self._set_metadata('language', values.get('lang'), 3)
elif tag == 'head':
self._in_head = True
elif tag == 'title':
self._in_title = True
self._title_parts = []
elif tag == 'meta':
identity = (
values.get('name') or values.get('property')
or values.get('itemprop') or ''
).lower()
content = values.get('content', '')
fields = {
'description': ('description', 3),
'og:description': ('description', 2),
'twitter:description': ('description', 1),
'author': ('author', 3),
'article:author': ('author', 2),
'date': ('date', 3),
'datepublished': ('date', 3),
'article:published_time': ('date', 3),
'datemodified': ('lastUpdated', 3),
'last-modified': ('lastUpdated', 3),
'article:modified_time': ('lastUpdated', 3),
'og:title': ('title', 2),
'twitter:title': ('title', 1),
}
if identity in fields:
key, priority = fields[identity]
self._set_metadata(key, content, priority)
elif tag == 'link' and 'canonical' in values.get('rel', '').lower().split():
canonical = values.get('href', '').strip()
if canonical:
parsed = urllib.parse.urlparse(canonical)
path = urllib.parse.unquote(parsed.path or '/')
self._set_metadata('slug', path.rstrip('/') or '/', 3)
elif tag == 'time':
identity = (values.get('itemprop') or values.get('class') or '').lower()
if 'datemodified' in identity:
self._set_metadata('lastUpdated', values.get('datetime'), 2)
elif 'datepublished' in identity or 'published' in identity:
self._set_metadata('date', values.get('datetime'), 2)
if tag == 'script' and values.get('type', '').lower() == 'application/ld+json':
self._json_ld_parts = []
if tag in self._SKIPPED_TAGS:
self._skip_depth += 1
return
if self._skip_depth or self._in_head:
return
if re.fullmatch(r'h[1-6]', tag):
self._newline(2)
self._parts.append('#' * int(tag[1]) + ' ')
elif tag == 'a':
self._parts.append('[')
self._anchors.append(values.get('href', ''))
elif tag in {'ul', 'ol'}:
self._lists.append(tag)
self._newline(1)
elif tag == 'li':
self._newline(1)
self._parts.append('1. ' if self._lists and self._lists[-1] == 'ol' else '- ')
elif tag == 'br':
self._newline(1)
elif tag == 'blockquote':
self._newline(2)
self._parts.append('> ')
elif tag in {'strong', 'b'}:
self._parts.append('**')
elif tag in {'em', 'i'}:
self._parts.append('*')
elif tag == 'table':
self._table_depth += 1
self._table_first_row = True
self._newline(2)
elif tag == 'tr' and self._table_depth:
self._table_row_cells = 0
self._newline(1)
elif tag in {'th', 'td'} and self._table_depth:
self._table_row_cells += 1
self._parts.append('| ')
elif tag in self._BLOCK_TAGS:
self._newline(2)
def handle_endtag(self, tag: str) -> None:
tag = tag.lower()
if tag == 'title':
self._in_title = False
self._set_metadata('title', ''.join(self._title_parts), 3)
elif tag == 'head':
self._in_head = False
if tag in self._SKIPPED_TAGS:
if tag == 'script' and self._json_ld_parts is not None:
self._extract_json_ld_metadata(''.join(self._json_ld_parts))
self._json_ld_parts = None
self._skip_depth = max(self._skip_depth - 1, 0)
return
if self._skip_depth or self._in_head:
return
if re.fullmatch(r'h[1-6]', tag) or tag in self._BLOCK_TAGS or tag == 'blockquote':
self._newline(2)
elif tag == 'a' and self._anchors:
self._parts.append(f']({self._anchors.pop()})')
elif tag in {'ul', 'ol'}:
if self._lists:
self._lists.pop()
self._newline(2)
elif tag == 'li':
self._newline(1)
elif tag in {'strong', 'b'}:
self._parts.append('**')
elif tag in {'em', 'i'}:
self._parts.append('*')
elif tag in {'th', 'td'} and self._table_depth:
self._parts.append(' ')
elif tag == 'tr' and self._table_depth:
self._parts.append('|\n')
if self._table_first_row and self._table_row_cells:
self._parts.append('| ' + ' | '.join(['---'] * self._table_row_cells) + ' |\n')
self._table_first_row = False
elif tag == 'table':
self._table_depth = max(self._table_depth - 1, 0)
self._newline(2)
def handle_data(self, data: str) -> None:
if self._json_ld_parts is not None:
self._json_ld_parts.append(data)
return
if self._in_title:
self._title_parts.append(data)
return
if self._skip_depth or self._in_head:
return
text = re.sub(r'\s+', ' ', data)
if text.strip():
self._parts.append(text)
def analysis_text(self) -> str:
lines = [line.strip() for line in ''.join(self._parts).splitlines()]
return re.sub(r'\n{3,}', '\n\n', '\n'.join(lines)).strip()
def extract_html_for_analysis(content: str) -> tuple[dict[str, Any], str]:
"""Return HTML metadata and normalized reader-visible analysis text."""
parser = _HTMLAnalysisParser()
parser.feed(content)
parser.close()
return dict(parser.metadata), parser.analysis_text()
def _detect_language(frontmatter: dict[str, Any], body: str) -> str:
"""Resolve a supported language profile without broad language guessing."""
declared = str(
frontmatter.get('lang')
or frontmatter.get('language')
or frontmatter.get('inLanguage')
or ''
).strip().lower()
if declared:
primary = re.split(r'[-_]', declared, maxsplit=1)[0]
return primary if primary in LANGUAGE_PROFILES else 'en'
strong_turkish_markers = len(re.findall(r'[ığşİĞŞ]', body))
letters = len(re.findall(r'[^\W\d_]', body, re.UNICODE))
if strong_turkish_markers >= 2 and strong_turkish_markers / max(letters, 1) >= 0.002:
return 'tr'
return 'en'
def _plain_text_for_analysis(content: str) -> str:
"""Remove non-prose payloads while retaining reader-visible prose."""
text = re.sub(r'```.*?```', '', content, flags=re.DOTALL)
text = re.sub(r'<(?:script|style|svg)\b.*?</(?:script|style|svg)\s*>', '', text,
flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r'^\s*\|.*\|\s*$', '', text, flags=re.MULTILINE)
text = re.sub(r'<[^>]+>', '', text)
text = re.sub(r'!\[[^\]\n]{0,300}\]\([^)\n]{0,1000}\)', '', text)
text = re.sub(r'\[([^\]\n]{1,300})\]\([^)\n]{1,1000}\)', r'\1', text)
text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
text = re.sub(r'^\s*(?:[-*+]|\d+\.)\s+', '', text, flags=re.MULTILINE)
text = re.sub(r'^\s*>\s?', '', text, flags=re.MULTILINE)
return re.sub(r'\n{3,}', '\n\n', text).strip()
# ---------------------------------------------------------------------------
# Heading analysis (extended from original)
# ---------------------------------------------------------------------------
def analyze_headings(content: str) -> dict[str, Any]:
"""Analyze heading structure and keyword placement."""
headings: list[dict[str, Any]] = []
for match in re.finditer(r'^(#{1,6})\s+(.+)$', content, re.MULTILINE):
level = len(match.group(1))
text = match.group(2).strip()
is_question = text.rstrip().endswith('?')
headings.append({
'level': level,
'text': text,
'is_question': is_question,
})
h1_count = sum(1 for h in headings if h['level'] == 1)
h2_count = sum(1 for h in headings if h['level'] == 2)
h3_count = sum(1 for h in headings if h['level'] == 3)
h2_questions = sum(1 for h in headings if h['level'] == 2 and h['is_question'])
question_ratio = h2_questions / h2_count if h2_count > 0 else 0
# Check for hierarchy skips
hierarchy_clean = True
prev_level = 0
for h in headings:
if h['level'] > prev_level + 1 and prev_level > 0:
hierarchy_clean = False
prev_level = h['level']
return {
'headings': headings,
'h1_count': h1_count,
'h2_count': h2_count,
'h3_count': h3_count,
'h2_question_count': h2_questions,
'h2_question_ratio': round(question_ratio, 2),
'hierarchy_clean': hierarchy_clean,
'total': len(headings),
}
# ---------------------------------------------------------------------------
# Paragraph analysis (kept from original)
# ---------------------------------------------------------------------------
def analyze_paragraphs(content: str) -> dict[str, Any]:
"""Analyze paragraph lengths."""
cleaned = re.sub(r'```.*?```', '', content, flags=re.DOTALL)
cleaned = re.sub(r'<[^>]+>', '', cleaned)
cleaned = re.sub(r'^#{1,6}\s+.*$', '', cleaned, flags=re.MULTILINE)
cleaned = re.sub(r'!\[[^\]\n]{0,300}\]\([^)\n]{0,1000}\)', '', cleaned)
paragraphs = [p.strip() for p in re.split(r'\n\s*\n', cleaned) if p.strip()]
word_counts: list[int] = []
over_150 = 0
over_200 = 0
in_range = 0 # 40-80 words (ideal paragraph range)
for p in paragraphs:
words = len(p.split())
if words < 5:
continue
word_counts.append(words)
if words > 200:
over_200 += 1
if words > 150:
over_150 += 1
if 40 <= words <= 80:
in_range += 1
total = len(word_counts)
avg = sum(word_counts) / total if total else 0
in_range_ratio = in_range / total if total else 0
return {
'total_paragraphs': total,
'avg_word_count': round(avg, 1),
'over_150_words': over_150,
'over_200_words': over_200,
# Backward-compatible aliases
'over_100_words': over_150,
'in_ideal_range': in_range,
'in_40_55_range': in_range,
'in_range_ratio': round(in_range_ratio, 2),
'max_word_count': max(word_counts) if word_counts else 0,
'total_word_count': sum(word_counts),
}
# ---------------------------------------------------------------------------
# Image analysis (extended from original)
# ---------------------------------------------------------------------------
def analyze_images(content: str) -> dict[str, Any]:
"""Count images and check alt text, formats, optimization signals."""
md_images = re.findall(r'!\[([^\]\n]{0,300})\]\(([^)\n]{1,1000})\)', content)
html_images: list[tuple[str, str]] = []
for tag in re.findall(r'<img\b[^>]*>', content, re.IGNORECASE):
src_match = re.search(r'\bsrc\s*=\s*["\']([^"\']+)["\']', tag, re.IGNORECASE)
if not src_match:
continue
alt_match = re.search(r'\balt\s*=\s*["\']([^"\']*)["\']', tag, re.IGNORECASE)
html_images.append((
alt_match.group(1) if alt_match else "",
src_match.group(1),
))
images: list[dict[str, Any]] = []
for alt, src in md_images:
ext = Path(src.split('?')[0]).suffix.lower()
images.append({
'src': src,
'has_alt': bool(alt.strip()),
'alt_length': len(alt.strip()),
'format': ext,
'source': 'pixabay' if 'pixabay' in src else
'unsplash' if 'unsplash' in src else
'pexels' if 'pexels' in src else 'other',
})
for alt, src in html_images:
has_alt = bool(alt.strip())
ext = Path(src.split('?')[0]).suffix.lower()
images.append({
'src': src,
'has_alt': has_alt,
'alt_length': len(alt.strip()),
'format': ext,
'source': 'pixabay' if 'pixabay' in src else
'unsplash' if 'unsplash' in src else
'pexels' if 'pexels' in src else 'other',
})
with_alt = sum(1 for img in images if img['has_alt'])
modern_formats = sum(1 for img in images if img.get('format') in ('.webp', '.avif', '.svg'))
return {
'count': len(images),
'with_alt_text': with_alt,
'without_alt_text': len(images) - with_alt,
'modern_format_count': modern_formats,
'formats': list(set(img.get('format', '') for img in images)),
'sources': {s: sum(1 for i in images if i['source'] == s)
for s in set(i['source'] for i in images)} if images else {},
}
# ---------------------------------------------------------------------------
# Chart analysis (kept from original)
# ---------------------------------------------------------------------------
def analyze_charts(content: str) -> dict[str, Any]:
"""Count SVG charts and check for type diversity."""
svg_count = len(re.findall(r'<svg\b', content, re.IGNORECASE))
figure_count = len(re.findall(r'<figure\b', content, re.IGNORECASE))
return {
'svg_count': svg_count,
'figure_count': figure_count,
'chart_count': max(svg_count, figure_count),
}
# ---------------------------------------------------------------------------
# Citation analysis (extended from original)
# ---------------------------------------------------------------------------
def _classify_source_tier(url: str) -> int:
"""Classify a URL into tier 1, 2, or 3."""
host = _hostname(url)
if not host:
return 3
for domain in TIER1_DOMAINS:
if _host_matches_domain(host, domain):
return 1
for domain in TIER2_DOMAINS:
if _host_matches_domain(host, domain):
return 2
return 3
def analyze_citations(content: str) -> dict[str, Any]:
"""Analyze statistics and their citations with tier classification."""
stat_patterns = re.findall(r'\d+\.?\d*%', content)
# Inline citations: [text](url)
inline_matches = re.findall(r'\[([^\]\n]{1,300})\]\((https?://[^)\n]{1,1000})\)', content)
citations_with_urls = [(text, url) for text, url in inline_matches]
# Parenthetical citations (Source Name, year)
paren_citations = re.findall(r'\(([^)]*(?:20\d{2})[^)]*)\)', content)
# Tier classification
tier_counts = {1: 0, 2: 0, 3: 0}
for _, url in citations_with_urls:
tier = _classify_source_tier(url)
tier_counts[tier] += 1
# Sourced vs unsourced stats
sourced_stats = 0
unsourced_stats = 0
for stat_value in stat_patterns:
pos = content.find(stat_value)
if pos >= 0:
context = content[pos:pos + 200]
if re.search(r'\[[^\]\n]{1,300}\]\(https?://', context) or re.search(r'\([^)\n]{0,200}20\d{2}[^)\n]{0,200}\)', context):
sourced_stats += 1
else:
unsourced_stats += 1
return {
'total_statistics': len(stat_patterns),
'sourced_statistics': sourced_stats,
'unsourced_statistics': unsourced_stats,
'inline_citations': len(citations_with_urls),
'paren_citations': len(paren_citations),
'unique_sources': len(set(url.lower() for _, url in citations_with_urls)),
'tier_counts': tier_counts,
}
# ---------------------------------------------------------------------------
# FAQ analysis (kept from original)
# ---------------------------------------------------------------------------
def analyze_faq(content: str) -> dict[str, Any]:
"""Check for FAQ section and schema."""
has_faq_section = bool(re.search(r'(?i)#{1,3}\s*(?:FAQ|Frequently Asked)', content))
has_faq_schema = bool(re.search(r'(?i)FAQSchema|FAQPage|faqpage', content))
faq_items = 0
if has_faq_section:
faq_match = re.search(r'(?i)#{1,3}\s*(?:FAQ|Frequently Asked).*', content, re.DOTALL)
if faq_match:
faq_text = faq_match.group()
faq_items = len(re.findall(r'^#{3,4}\s+.+\?', faq_text, re.MULTILINE))
return {
'has_faq_section': has_faq_section,
'has_faq_schema': has_faq_schema,
'faq_item_count': faq_items,
}
# ---------------------------------------------------------------------------
# Freshness analysis (kept from original)
# ---------------------------------------------------------------------------
def analyze_freshness(frontmatter: dict[str, Any]) -> dict[str, Any]:
"""Check freshness signals."""
return {
'has_date': 'date' in frontmatter,
'has_last_updated': 'lastUpdated' in frontmatter or 'last_updated' in frontmatter,
'date': frontmatter.get('date', ''),
'last_updated': frontmatter.get('lastUpdated', frontmatter.get('last_updated', '')),
}
# ---------------------------------------------------------------------------
# Self-promotion analysis (kept from original)
# ---------------------------------------------------------------------------
def analyze_self_promotion(content: str, brand_name: str = '') -> dict[str, Any]:
"""Check self-promotion levels."""
promo_patterns = [
r'(?i)at \w+,\s+we',
r'(?i)our (?:team|company|product|platform|solution)',
r'(?i)we (?:offer|provide|deliver|help|specialize)',
]
promo_count = sum(len(re.findall(p, content)) for p in promo_patterns)
return {
'self_promotion_patterns': promo_count,
'exceeds_limit': promo_count > 1,
}
# ---------------------------------------------------------------------------
# NEW: Readability analysis (graceful degradation)
# ---------------------------------------------------------------------------
def analyze_readability(text: str, language: str = 'en') -> dict[str, Any]:
"""Compute readability metrics using textstat if available, else estimate."""
words = re.findall(r"[^\W\d_]+(?:['’][^\W\d_]+)?", text, re.UNICODE)
word_count = len(words)
sentences = re.findall(r'[.!?]+', text)
sentence_count = len(sentences) if sentences else 1
avg_sentence_len = word_count / sentence_count
if LANGUAGE_PROFILES.get(language, LANGUAGE_PROFILES['en'])["readability_model"] == 'atesman':
vowel_count = sum(len(re.findall(r'[aeıioöuü]', word.lower())) for word in words)
mean_syllables = vowel_count / max(word_count, 1)
score = 198.825 - 40.175 * mean_syllables - 2.610 * avg_sentence_len
score = max(0.0, min(100.0, score))
return {
'reading_model': 'atesman',
'reading_ease': round(score, 1),
'atesman_reading_ease': round(score, 1),
'reading_time_minutes': round(word_count / 238, 1),
'avg_sentence_length': round(avg_sentence_len, 1),
'mean_syllables_per_word': round(mean_syllables, 2),
'estimated': False,
}
if HAS_TEXTSTAT:
fre = textstat.flesch_reading_ease(text)
fkg = textstat.flesch_kincaid_grade(text)
fog = textstat.gunning_fog(text)
try:
reading_time = round(textstat.reading_time(text, ms_per_char=14.69) / 60, 1)
except Exception:
reading_time = round(word_count / 238, 1)
return {
'reading_model': 'flesch',
'reading_ease': round(fre, 1),
'flesch_reading_ease': round(fre, 1),
'flesch_kincaid_grade': round(fkg, 1),
'gunning_fog': round(fog, 1),
'reading_time_minutes': reading_time,
'avg_sentence_length': round(avg_sentence_len, 1),
'estimated': False,
}
else:
# Rough Flesch estimate: 206.835 - 1.015*(words/sentences) - 84.6*(syllables/words)
avg_word_len = len(text) / max(word_count, 1)
est_syllable_ratio = avg_word_len / 4.7 # crude approximation
fre = max(0, 206.835 - 1.015 * avg_sentence_len - 84.6 * est_syllable_ratio)
return {
'reading_model': 'flesch-estimate',
'reading_ease': round(fre, 1),
'flesch_reading_ease': round(fre, 1),
'reading_time_minutes': round(word_count / 238, 1),
'avg_sentence_length': round(avg_sentence_len, 1),
'estimated': True,
}
# ---------------------------------------------------------------------------
# NEW: Sentence analysis
# ---------------------------------------------------------------------------
def analyze_sentences(text: str) -> dict[str, Any]:
"""Analyze sentence lengths, burstiness (variance), and engagement."""
sentences = re.split(r'(?<=[.!?])\s+', text)
lengths = [len(s.split()) for s in sentences if len(s.split()) > 2]
if not lengths:
return {
'count': 0,
'avg_length': 0,
'max_length': 0,
'burstiness': 0.0,
'std_dev': 0.0,
'very_long_count': 0,
}
avg = sum(lengths) / len(lengths)
std_dev = (sum((length - avg) ** 2 for length in lengths) / len(lengths)) ** 0.5
burstiness = std_dev / avg if avg > 0 else 0
very_long = sum(1 for length in lengths if length > 40)
over_20 = sum(1 for length in lengths if length > 20)
over_25 = sum(1 for length in lengths if length > 25)
total = len(lengths)
return {
'count': total,
'avg_length': round(avg, 1),
'max_length': max(lengths),
'burstiness': round(burstiness, 2),
'std_dev': round(std_dev, 1),
'very_long_count': very_long,
'over_20_count': over_20,
'over_20_pct': round(over_20 / total * 100, 1) if total else 0,
'over_25_count': over_25,
}
# ---------------------------------------------------------------------------
# Editorial style diagnostics
# ---------------------------------------------------------------------------
def analyze_ai_signals(text: str, sentences_info: dict[str, Any]) -> dict[str, Any]:
"""Return non-scoring editorial style diagnostics.
Phrase frequency, type-token ratio, and sentence-length variance cannot
determine authorship. The legacy keys remain for output compatibility, but
the analyzer never converts these observations into an AI probability or a
quality-gate decision.
"""
found_phrases: list[dict[str, Any]] = []
lower_text = text.lower()
for phrase in AI_PHRASES:
count = lower_text.count(phrase)
if count > 0:
found_phrases.append({'phrase': phrase, 'count': count})
words = text.split()
unique = len(set(w.lower() for w in words))
ttr = unique / len(words) if words else 0
return {
'ai_phrases_found': found_phrases,
'ai_phrase_count': sum(p['count'] for p in found_phrases),
'vocabulary_diversity_ttr': round(ttr, 3),
'burstiness': sentences_info.get('burstiness', 0),
'likely_ai': None,
'editorial_style_only': True,
'not_an_authorship_classifier': True,
}
# ---------------------------------------------------------------------------
# NEW: Passive voice estimation
# ---------------------------------------------------------------------------
def analyze_passive_voice(text: str) -> dict[str, Any]:
"""Estimate passive voice percentage using regex heuristics."""
sentences = re.split(r'(?<=[.!?])\s+', text)
sentences = [s for s in sentences if len(s.split()) > 2]
if not sentences:
return {'passive_count': 0, 'total_sentences': 0, 'passive_pct': 0.0}
passive_pattern = re.compile(
r'\b(was|were|been|being|is|are|am|get|got|gets|getting)\s+'
r'(\w+ly\s+)?' # optional adverb
r'(\w+ed|written|spoken|taken|given|made|done|seen|known|shown|built|sent|found|held|told|left|run|set|kept|brought|thought|put)\b',
re.IGNORECASE,
)
passive_count = sum(1 for s in sentences if passive_pattern.search(s))
return {
'passive_count': passive_count,
'total_sentences': len(sentences),
'passive_pct': round(passive_count / len(sentences) * 100, 1),
}
# ---------------------------------------------------------------------------
# NEW: Transition word analysis
# ---------------------------------------------------------------------------
def analyze_transition_words(text: str) -> dict[str, Any]:
"""Measure percentage of sentences containing transition words."""
sentences = re.split(r'(?<=[.!?])\s+', text)
sentences = [s for s in sentences if len(s.split()) > 2]
if not sentences:
return {'transition_count': 0, 'total_sentences': 0, 'transition_pct': 0.0}
lower_sentences = [s.lower() for s in sentences]
transition_count = 0
for s in lower_sentences:
for tw in TRANSITION_WORDS:
if tw in s:
transition_count += 1
break # count each sentence once
return {
'transition_count': transition_count,
'total_sentences': len(sentences),
'transition_pct': round(transition_count / len(sentences) * 100, 1),
}
# ---------------------------------------------------------------------------
# NEW: AI trigger word detection
# ---------------------------------------------------------------------------
def analyze_ai_trigger_words(text: str) -> dict[str, Any]:
"""Count AI trigger words per 1,000 words."""
words = text.split()
word_count = len(words)
if word_count == 0:
return {'trigger_count': 0, 'per_1k': 0.0, 'found': []}
lower_text = text.lower()
found: list[dict[str, Any]] = []
total = 0
for tw in AI_TRIGGER_WORDS:
count = len(re.findall(r'\b' + re.escape(tw) + r'\b', lower_text))
if count > 0:
found.append({'word': tw, 'count': count})
total += count
per_1k = round(total / word_count * 1000, 1)
return {
'trigger_count': total,
'per_1k': per_1k,
'found': found,
}
# ---------------------------------------------------------------------------
# NEW: Schema / structured data detection (graceful degradation)
# ---------------------------------------------------------------------------
def analyze_schema(content: str) -> dict[str, Any]:
"""Detect JSON-LD schema markup in content."""
schemas: list[str] = []
def collect_types(value: Any) -> None:
if isinstance(value, dict):
schema_type = value.get('@type')
if isinstance(schema_type, str):
schemas.append(schema_type)
elif isinstance(schema_type, list):
schemas.extend(item for item in schema_type if isinstance(item, str))
for child in value.values():
if isinstance(child, (dict, list)):
collect_types(child)
elif isinstance(value, list):
for item in value:
collect_types(item)
if HAS_BS4:
try:
soup = BeautifulSoup(content, 'html.parser')
for script in soup.find_all('script', type='application/ld+json'):
try:
data = json.loads(script.string)
collect_types(data)
except (json.JSONDecodeError, TypeError, RecursionError):
pass
except Exception:
pass
else:
# Fallback: parse JSON-LD script bodies without an HTML dependency.
for script_body in re.findall(
r'<script\b[^>]*type=["\']application/ld\+json["\'][^>]*>(.*?)</script\s*>',
content,
re.DOTALL | re.IGNORECASE,
):
try:
collect_types(json.loads(script_body))
except (json.JSONDecodeError, TypeError, RecursionError):
for match in re.findall(r'"@type"\s*:\s*"([^"]+)"', script_body):
schemas.append(match)
return {
'schemas_found': schemas,
'schema_count': len(schemas),
'has_blogposting': 'BlogPosting' in schemas or 'Article' in schemas,
'has_faqpage': 'FAQPage' in schemas,
'has_person': 'Person' in schemas,
'has_organization': 'Organization' in schemas,
'has_breadcrumblist': 'BreadcrumbList' in schemas,
}
# ---------------------------------------------------------------------------
# NEW: Link analysis
# ---------------------------------------------------------------------------
def analyze_links(content: str) -> dict[str, Any]:
"""Analyze internal and external links, anchor quality, and tiers."""
# Internal links: relative paths (not starting with http or /)
internal = re.findall(r'\[([^\]\n]{1,300})\]\((?!https?://|#)([^)\n]{1,1000})\)', content)
# External links
external = re.findall(r'\[([^\]\n]{1,300})\]\((https?://[^)\n]{1,1000})\)', content)
bad_anchor_keywords = {'click here', 'read more', 'this article', 'here', 'link', 'this'}
bad_anchors = [a for a, _ in internal + external if a.lower().strip() in bad_anchor_keywords]
# Tier classification for external links
tier_counts = {1: 0, 2: 0, 3: 0}
for _, url in external:
tier = _classify_source_tier(url)
tier_counts[tier] += 1
return {
'internal_count': len(internal),
'external_count': len(external),
'total_links': len(internal) + len(external),
'bad_anchor_texts': bad_anchors,
'external_tier_counts': tier_counts,
}
# ---------------------------------------------------------------------------
# NEW: Originality markers
# ---------------------------------------------------------------------------
def analyze_originality(content: str, language: str = 'en') -> dict[str, Any]:
"""Detect distinctive evidence and first-hand claims without inferring truth."""
markers: list[str] = []
if re.search(r'(?:\[|<!--\s*)ORIGINAL DATA(?:\]|(?:\s*:.*)?-->)', content, re.IGNORECASE):
markers.append('original_data_tag')
if re.search(r'(?:\[|<!--\s*)PERSONAL EXPERIENCE(?:\]|(?:\s*:.*)?-->)', content, re.IGNORECASE):
markers.append('personal_experience_tag')
if re.search(r'(?:\[|<!--\s*)UNIQUE INSIGHT(?:\]|(?:\s*:.*)?-->)', content, re.IGNORECASE):
markers.append('unique_insight_tag')
profile = LANGUAGE_PROFILES.get(language, LANGUAGE_PROFILES['en'])
first_person_patterns = profile['first_person_patterns']
first_person_count = 0
for pattern in first_person_patterns:
first_person_count += len(re.findall(pattern, content, re.IGNORECASE))
if first_person_count > 0:
markers.append('first_person_experience')
methodology_patterns = profile['methodology_patterns']
methodology_count = sum(
len(re.findall(pattern, content, re.IGNORECASE))
for pattern in methodology_patterns
)
evidence_marker_count = sum(
1 for marker in markers
if marker in {'original_data_tag', 'personal_experience_tag', 'unique_insight_tag'}
)
unsupported_experience_claims = 0
for paragraph in re.split(r'\n\s*\n', content):
has_claim = any(re.search(pattern, paragraph, re.IGNORECASE)
for pattern in first_person_patterns)
has_method = any(re.search(pattern, paragraph, re.IGNORECASE)
for pattern in methodology_patterns)
has_support = bool(re.search(
r'(?:ORIGINAL DATA|PERSONAL EXPERIENCE|UNIQUE INSIGHT|\d|https?://)',
paragraph,
re.IGNORECASE,
))
if has_claim and not (has_method or has_support):
unsupported_experience_claims += 1
return {
'markers': markers,
'marker_count': len(markers),
'first_person_count': first_person_count,
'evidence_marker_count': evidence_marker_count,
'methodology_count': methodology_count,
'unsupported_experience_claims': unsupported_experience_claims,
}
# ---------------------------------------------------------------------------
# NEW: Engagement elements
# ---------------------------------------------------------------------------
def analyze_engagement(content: str) -> dict[str, Any]:
"""Detect questions in body text, examples, call-to-action patterns."""
# Questions in body (not in headings)
body_lines = [line for line in content.split('\n') if not line.strip().startswith('#')]
body_text = '\n'.join(body_lines)
questions_in_text = len(re.findall(r'[^#]\?', body_text))
# Example markers
example_patterns = [
r'(?i)\bfor example\b', r'(?i)\bfor instance\b', r'(?i)\bsuch as\b',
r'(?i)\bconsider\b', r'(?i)\blet\'s say\b', r'(?i)\bimagine\b',
r'(?i)\bhere\'s (?:an|a) example\b',
]
example_count = sum(len(re.findall(p, content)) for p in example_patterns)
return {
'questions_in_text': questions_in_text,
'example_count': example_count,
}
# ---------------------------------------------------------------------------
# NEW: AI citation readiness
# ---------------------------------------------------------------------------
def analyze_ai_citation_readiness(content: str, headings_info: dict[str, Any],
faq_info: dict[str, Any],
language: str = 'en') -> dict[str, Any]:
"""Assess evidence-backed reuse readiness without fixed passage lengths."""
section_pattern = re.compile(
r'^##\s+(.+?)\s*$\n(.*?)(?=^##\s+|\Z)',
re.MULTILINE | re.DOTALL,
)
sections = section_pattern.findall(content)
evidence_backed_sections = 0
self_contained_sections = 0
for heading, section in sections:
has_source = bool(re.search(r'\[[^\]\n]{1,300}\]\(https?://[^)\n]{1,1000}\)', section))
has_evidence_marker = bool(re.search(
r'(?:ORIGINAL DATA|PERSONAL EXPERIENCE|UNIQUE INSIGHT)',
section,
re.IGNORECASE,
))
has_definition = bool(re.search(
r'\*\*[^*]+\*\*\s*(?:is|are|refers to|means|adalah|merupakan)',
section,
re.IGNORECASE,
))
has_specific_support = bool(re.search(r'\b\d+(?:\.\d+)?%?\b', section))
if has_source or has_evidence_marker:
evidence_backed_sections += 1
if section.strip() and (has_source or has_evidence_marker or has_definition) and (
has_specific_support or has_definition
):
self_contained_sections += 1
# Q&A detection: question headings followed by direct answers
qa_pairs = 0
lines = content.split('\n')
for i, line in enumerate(lines):
if re.match(r'^#{2,4}\s+.+\?', line):
# Check if next non-empty line starts with a direct statement
for j in range(i + 1, min(i + 5, len(lines))):
if lines[j].strip():
if not lines[j].strip().startswith('#'):
qa_pairs += 1
break
# Entity clarity: detect defined terms (bold terms followed by explanations)
entity_definitions = len(re.findall(
r'\*\*[^*]+\*\*\s*(?:is|are|refers to|means|adalah|merupakan)',
content,
))
# Extraction-friendly structures
profile = LANGUAGE_PROFILES.get(language, LANGUAGE_PROFILES['en'])
summary_pattern = '|'.join(profile['summary_labels'])
has_tldr = bool(re.search(rf'(?i)(?:{summary_pattern})', content))
table_count = len(re.findall(r'^\|.+\|$', content, re.MULTILINE))
list_count = len(re.findall(r'^[\s]*[-*+]\s', content, re.MULTILINE))
# AI crawler accessibility: inspect actual meta / robots directives only.
has_robots_meta = bool(re.search(
r'(?is)<meta[^>]+name=["\']robots["\'][^>]+content=["\'][^"\']*(?:noindex|noai)[^"\']*["\']',
content,
))
intro = re.split(r'^##\s+', content, maxsplit=1, flags=re.MULTILINE)[0]
purpose_statement = bool(re.search(r'[A-Za-z]{3,}', intro))
relevant_media_count = (
len(re.findall(r'!\[[^\]\n]{1,300}\]\([^)\n]{1,1000}\)', content))
+ len(re.findall(r'<(?:img|figure|svg)\b', content, re.IGNORECASE))
)
return {
# Compatibility alias: now counts evidence-backed, self-contained
# sections rather than passages in a prescribed word-count band.
'citable_passages': self_contained_sections,
'self_contained_sections': self_contained_sections,
'evidence_backed_sections': evidence_backed_sections,
'section_count': len(sections),
'qa_pairs': qa_pairs,
'entity_definitions': entity_definitions,
'has_tldr': has_tldr,
'table_count': table_count,
'list_count': list_count,
'has_robots_restriction': has_robots_meta,
'purpose_statement': purpose_statement,
'relevant_media_count': relevant_media_count,
}
# ---------------------------------------------------------------------------
# NEW: OG / social meta tags
# ---------------------------------------------------------------------------
def analyze_social_meta(content: str, frontmatter: dict[str, Any]) -> dict[str, Any]:
"""Detect Open Graph and social media meta tags."""
og_tags = re.findall(r'(?:og:|twitter:)\w+', content)
# Also check frontmatter for common social fields
social_fields = ['image', 'og_image', 'ogImage', 'twitter_image',
'social_image', 'thumbnail', 'cover']
has_social_image = any(f in frontmatter for f in social_fields)
return {
'og_tags_found': len(og_tags),
'has_social_image': has_social_image,
'social_fields_in_frontmatter': [f for f in social_fields if f in frontmatter],
}
# ---------------------------------------------------------------------------
# NEW: Structured data signals (tables, lists)
# ---------------------------------------------------------------------------
def analyze_structured_data(content: str) -> dict[str, Any]:
"""Detect tables, ordered/unordered lists, and other structured elements."""
table_rows = len(re.findall(r'^\|.+\|$', content, re.MULTILINE))
tables = 0
if table_rows > 0:
# Estimate number of tables (each table has at least a header separator row)
tables = len(re.findall(r'^\|[-:| ]+\|$', content, re.MULTILINE))
unordered_items = len(re.findall(r'^[\s]*[-*+]\s', content, re.MULTILINE))
ordered_items = len(re.findall(r'^[\s]*\d+\.\s', content, re.MULTILINE))
code_blocks = len(re.findall(r'```', content)) // 2
blockquotes = len(re.findall(r'^>\s', content, re.MULTILINE))
return {
'table_count': tables,
'table_rows': table_rows,
'unordered_list_items': unordered_items,
'ordered_list_items': ordered_items,
'code_blocks': code_blocks,
'blockquotes': blockquotes,
}
# ---------------------------------------------------------------------------
# Scoring: 5-category, 100-point system
# ---------------------------------------------------------------------------
_TOPIC_STOPWORDS = {
'about', 'after', 'also', 'and', 'are', 'but', 'for', 'from', 'have',
'how', 'into', 'its', 'not', 'that', 'the', 'their', 'this', 'with',
'what', 'when', 'where', 'which', 'why', 'your',
}
def _topic_terms(value: str) -> set[str]:
"""Return normalized topic terms for purpose-consistency checks."""
return {
token
for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9'-]*", value.lower())
if token not in _TOPIC_STOPWORDS and not token.isdigit()
}
def _detect_content_type(frontmatter: dict[str, Any], headings_info: dict[str, Any],
content: str) -> str:
"""Guess the content type from frontmatter or content patterns."""
category = frontmatter.get('category', '').lower()
title = frontmatter.get('title', '').lower()
content_type = frontmatter.get('type', '').lower()
if content_type and content_type in CONTENT_TYPE_BENCHMARKS:
return content_type
if 'guide' in title or 'guide' in category:
return 'guide'
if 'how to' in title or 'how-to' in category:
return 'how-to'
if re.search(r'^\d+\s', title) or 'listicle' in category:
return 'listicle'
if 'review' in title or 'review' in category:
return 'review'
if 'case study' in title or 'case-study' in category:
return 'case-study'
if 'opinion' in category:
return 'opinion'
if 'news' in category:
return 'news'
return 'default'
def calculate_score(analysis: dict[str, Any]) -> dict[str, Any]:
"""Calculate the 5-category, 100-point quality score."""
issues: list[dict[str, Any]] = []
category_details: dict[str, dict[str, Any]] = {}
# ===================================================================
# CONTENT QUALITY (30 pts)
# ===================================================================
cq = 0
cq_breakdown: dict[str, Any] = {}
# Coverage / comprehensiveness: 7 pts. Word count is reported for context,
# never used as a quality target or scoring shortcut.
headings = analysis['headings']
citations = analysis['citations']
engagement = analysis['engagement']
depth_score = 0
if headings['h2_count'] >= 1:
depth_score += 2
if headings['h2_count'] >= 3:
depth_score += 1
if citations['inline_citations'] + citations['paren_citations'] >= 1:
depth_score += 2
if engagement['example_count'] >= 1:
depth_score += 1
if analysis['structured_data']['table_count'] >= 1 or (
analysis['structured_data']['unordered_list_items']
+ analysis['structured_data']['ordered_list_items']
) >= 3:
depth_score += 1
if depth_score < 3:
issues.append({
'category': 'content',
'severity': 'high',
'issue': 'Coverage is thin: add missing subtopics, evidence, or useful examples for the reader intent',
})
cq += depth_score
cq_breakdown['depth'] = depth_score
# Readability: 7 pts
# Default band: Flesch Ease 60-70 (Grade 7-8)
# Persona bands: Consumer 60-80, Professional 50-60, Technical 30-50
readability = analysis['readability']
reading_ease = readability.get(
'reading_ease', readability.get('flesch_reading_ease', 50)
)
reading_model = readability.get('reading_model', 'flesch')
if reading_model == 'atesman' and 50 <= reading_ease <= 69:
read_score = 7
elif reading_model == 'atesman' and 30 <= reading_ease <= 89:
read_score = 5
elif reading_model == 'atesman' and 0 <= reading_ease <= 100:
read_score = 3
elif 60 <= reading_ease <= 70:
read_score = 7
elif 55 <= reading_ease <= 75:
read_score = 5
elif 45 <= reading_ease <= 80:
read_score = 3
else:
read_score = 1
issues.append({'category': 'content', 'severity': 'medium',
'issue': f'Flesch reading ease ({reading_ease}) outside acceptable range (55-75)'})
cq += read_score
cq_breakdown['readability'] = read_score
# Originality / differentiated evidence: 5 pts
orig = analysis['originality']
orig_score = min(orig.get('evidence_marker_count', 0) * 2, 4)
if citations['unique_sources'] >= 2 and engagement['example_count'] >= 1:
# Neutral explainers can demonstrate value through synthesis and examples.
orig_score = max(orig_score, 3)
if orig.get('methodology_count', 0) > 0:
# A disclosed method is a bonus on top of the synthesis floor and must
# never replace it (disclosing sources must not lower the score).
orig_score = min(5, orig_score + 1)
if orig_score == 0:
issues.append({'category': 'content', 'severity': 'medium',
'issue': 'No differentiated evidence or analysis found - add original data, a sourced synthesis, a case example, or a clearly labeled unique insight'})
if orig.get('unsupported_experience_claims', 0) > 0:
issues.append({
'category': 'content',
'severity': 'high',
'issue': 'First-hand testing claims need supporting methodology, measurements, or evidence',
})
cq += orig_score
cq_breakdown['originality'] = orig_score
# Logical structure: 4 pts
struct_score = 0
if headings['h2_count'] >= 3:
struct_score += 2
elif headings['h2_count'] >= 1:
struct_score += 1
else:
issues.append({'category': 'content', 'severity': 'high',
'issue': 'No H2 headings - add section headings for structure'})
if headings['hierarchy_clean']:
struct_score += 1
else:
issues.append({'category': 'content', 'severity': 'medium',
'issue': 'Heading hierarchy has skips (e.g., H2 to H4)'})
# A single H1 provides a clear document-level topic without relying on
# paragraph-length thresholds.
if headings['h1_count'] == 1:
struct_score += 1
cq += struct_score
cq_breakdown['structure'] = struct_score
# Reader utility elements: 4 pts. Questions are not a scoring requirement.
eng_score = 0
if engagement['example_count'] >= 2:
eng_score += 2
elif engagement['example_count'] >= 1:
eng_score += 1
ai_ready = analysis['ai_citation_readiness']
if ai_ready['has_tldr']:
eng_score += 1
if analysis['structured_data']['table_count'] >= 1:
eng_score += 1
elif analysis['structured_data']['unordered_list_items'] + analysis['structured_data']['ordered_list_items'] >= 3:
eng_score += 1
eng_score = min(eng_score, 4)
if eng_score < 2:
issues.append({'category': 'content', 'severity': 'low',
'issue': 'Reader utility is limited - add a relevant example, summary, table, or decision aid when useful'})
cq += eng_score
cq_breakdown['engagement'] = eng_score
# Grammar / clarity: 3 pts. Authorship-style diagnostics never affect score.
sentences = analysis['sentences']
transitions = analysis.get('transition_words', {})
gram_score = 1 if sentences['count'] > 0 else 0
if sentences['very_long_count'] == 0:
gram_score += 1
if sentences['very_long_count'] > 0:
issues.append({'category': 'content', 'severity': 'low',
'issue': f'{sentences["very_long_count"]} sentences over 40 words - consider splitting'})
transition_pct = transitions.get('transition_pct', 0)
if sentences['count'] > 0 and 8 <= sentences['avg_length'] <= 30:
gram_score += 1
if transition_pct > 50:
issues.append({'category': 'content', 'severity': 'low',
'issue': f'Transition words occur in {transition_pct}% of sentences - review for repetitive connective phrasing'})
gram_score = min(gram_score, 3)
cq += gram_score
cq_breakdown['grammar_antipattern'] = gram_score
cq = min(cq, 30)
category_details['content_quality'] = {'score': cq, 'max': 30, 'breakdown': cq_breakdown}
# ===================================================================
# SEO OPTIMIZATION (25 pts)
# ===================================================================
seo = 0
seo_breakdown: dict[str, Any] = {}
fm = analysis['frontmatter']
# Title clarity and purpose fit: 4 pts. Character bands, sentiment, power
# words, and exact-match keyword placement do not earn points.
title = fm.get('title', '')
title_score = 0
body = analysis.get('_body_text', '')
heading_text = ' '.join(h['text'] for h in headings['headings'])
title_terms = _topic_terms(title)
body_terms = _topic_terms(body)
heading_terms = _topic_terms(heading_text)
if title:
title_score += 1
if title.strip().lower() not in {'home', 'blog', 'post', 'untitled'}:
title_score += 1
if title_terms & body_terms:
title_score += 1
if title_terms & heading_terms:
title_score += 1
if not title:
issues.append({'category': 'seo', 'severity': 'high',
'issue': 'Missing title in frontmatter'})
elif not (title_terms & body_terms):
issues.append({'category': 'seo', 'severity': 'medium',
'issue': 'Title does not clearly match the visible page topic'})
seo += title_score
seo_breakdown['title'] = title_score
# Heading hierarchy and reader navigation: 5 pts.
heading_score = 0
if headings['h1_count'] == 1:
heading_score += 2
elif headings['h1_count'] == 0 and title:
heading_score += 2 # Frontmatter title can serve as the document H1.
if headings['total'] > 0 and headings['hierarchy_clean']:
heading_score += 2
heading_values = [h['text'].strip().lower() for h in headings['headings']]
if heading_values and all(heading_values) and len(heading_values) == len(set(heading_values)):
heading_score += 1
heading_score = min(heading_score, 5)
seo += heading_score
seo_breakdown['headings'] = heading_score
# Semantic topic consistency: 4 pts. Keep the compatibility field name,
# but ignore exact-match placement quotas.
keyword_score = 0
if title_terms & body_terms:
keyword_score += 2
if title_terms & heading_terms:
keyword_score += 1
if heading_terms & body_terms:
keyword_score += 1
keyword_score = min(keyword_score, 4)
seo += keyword_score
seo_breakdown['keyword_placement'] = keyword_score
# Internal linking (3-10 contextual): 4 pts
links = analysis['links']
int_score = 0
ic = links['internal_count']
if 3 <= ic <= 10:
int_score = 4
elif ic >= 1:
int_score = 2
else:
issues.append({'category': 'seo', 'severity': 'high',
'issue': 'No internal links - add 3-10 contextual internal links'})
if links['bad_anchor_texts']:
int_score = max(int_score - 1, 0)
issues.append({'category': 'seo', 'severity': 'low',
'issue': f'Bad anchor texts found: {links["bad_anchor_texts"]}'})
seo += int_score
seo_breakdown['internal_linking'] = int_score
# Meta description accuracy and visible-content consistency: 3 pts.
desc = fm.get('description', fm.get('meta_description', ''))
meta_score = 0
if desc:
meta_score += 1
desc_terms = _topic_terms(desc)
if desc_terms & (title_terms | heading_terms):
meta_score += 1
if desc_terms & body_terms:
meta_score += 1
else:
issues.append({'category': 'seo', 'severity': 'high',
'issue': 'Missing meta description in frontmatter'})
seo += meta_score
seo_breakdown['meta_description'] = meta_score
# External linking (tier 1-3): 2 pts
ext_score = 0
if links['external_count'] >= 2:
ext_score += 1
tier_ext = links.get('external_tier_counts', {})
if tier_ext.get(1, 0) >= 1 or tier_ext.get(2, 0) >= 1:
ext_score += 1
seo += ext_score
seo_breakdown['external_linking'] = ext_score
# URL structure: 3 pts (from frontmatter slug)
slug = fm.get('slug', fm.get('url', ''))
url_score = 0
if slug:
if len(slug) <= 60:
url_score += 1
if '-' in slug and ' ' not in slug:
url_score += 1
if not re.search(r'\d{8,}', slug): # No long numeric strings
url_score += 1
else:
url_score = 1 # Partial credit; many static site generators auto-generate
url_score = min(url_score, 3)
seo += url_score
seo_breakdown['url_structure'] = url_score
seo = min(seo, 25)
category_details['seo_optimization'] = {'score': seo, 'max': 25, 'breakdown': seo_breakdown}
# ===================================================================
# E-E-A-T SIGNALS (15 pts)
# ===================================================================
eeat = 0
eeat_breakdown: dict[str, Any] = {}
# Author attribution: 4 pts
author = fm.get('author', fm.get('authors', ''))
author_score = 0
if author and author.lower() not in ('admin', 'administrator', 'staff', 'team', ''):
author_score = 4
elif author:
author_score = 1
issues.append({'category': 'eeat', 'severity': 'medium',
'issue': f'Generic author name "{author}" - use a real person name'})
else:
issues.append({'category': 'eeat', 'severity': 'high',
'issue': 'No author attribution in frontmatter'})
eeat += author_score
eeat_breakdown['author'] = author_score
# Source citations: 4 pts (tier-aware)
cit = analysis['citations']
cit_score = 0
total_citations = cit['inline_citations'] + cit['paren_citations']
if total_citations >= 5:
cit_score += 2
elif total_citations >= 2:
cit_score += 1
# Tier bonus
tier_c = cit.get('tier_counts', {})
if tier_c.get(1, 0) >= 1:
cit_score += 2
elif tier_c.get(2, 0) >= 1:
cit_score += 1
cit_score = min(cit_score, 4)
if total_citations == 0:
issues.append({'category': 'eeat', 'severity': 'high',
'issue': 'No source citations - add inline citations to credible sources'})
eeat += cit_score
eeat_breakdown['citations'] = cit_score
# Trust indicators: 4 pts (about/contact links, editorial mentions)
trust_score = 0
body = analysis.get('_body_text', '')
language = analysis.get('language', 'en')
profile = LANGUAGE_PROFILES.get(language, LANGUAGE_PROFILES['en'])
if any(re.search(pattern, body, re.IGNORECASE)
for pattern in profile['about_patterns']):
trust_score += 2
if any(re.search(pattern, body, re.IGNORECASE)
for pattern in profile['contact_patterns']):
trust_score += 1
if re.search(r'(?i)\b(?:editorial|reviewed by|fact.?check|editor)\b', body):
trust_score += 1
trust_score = min(trust_score, 4)
eeat += trust_score
eeat_breakdown['trust'] = trust_score
# Evidence basis: 3 pts. Never require or invent first-person experience.
orig = analysis['originality']
exp_score = 0
if orig.get('methodology_count', 0) >= 1 and orig.get('evidence_marker_count', 0) >= 1:
exp_score = 3
elif cit['unique_sources'] >= 3 and cit['unsourced_statistics'] == 0:
exp_score = 3
elif cit['unique_sources'] >= 1:
exp_score = 2
elif orig.get('evidence_marker_count', 0) >= 1:
exp_score = 1
if exp_score == 0:
issues.append({'category': 'eeat', 'severity': 'medium',
'issue': 'Evidence basis is unclear - add verifiable sources, transparent methodology, or clearly supported original material'})
eeat += exp_score
eeat_breakdown['experience'] = exp_score
eeat = min(eeat, 15)
category_details['eeat_signals'] = {'score': eeat, 'max': 15, 'breakdown': eeat_breakdown}
# ===================================================================
# TECHNICAL ELEMENTS (15 pts)
# ===================================================================
tech = 0
tech_breakdown: dict[str, Any] = {}
# Schema markup (JSON-LD): 4 pts
schema = analysis['schema']
schema_score = 0
if schema['has_blogposting']:
schema_score += 2
if schema['has_person']:
schema_score += 1
if schema.get('has_organization') or schema.get('has_breadcrumblist'):
schema_score += 1
if schema['schema_count'] == 0:
# Check if there are any schema signals at all
if re.search(r'(?i)json-ld|structured.?data|schema\.org', analysis.get('_raw_content', '')):
schema_score = 1
schema_score = min(schema_score, 4)
if schema_score == 0:
issues.append({'category': 'technical', 'severity': 'medium',
'issue': 'No JSON-LD schema markup detected - add BlogPosting schema'})
tech += schema_score
tech_breakdown['schema'] = schema_score
# Image optimization (alt text, formats): 3 pts
images = analysis['images']
img_score = 0
if images['count'] > 0:
alt_ratio = images['with_alt_text'] / images['count']
if alt_ratio == 1.0:
img_score += 2
elif alt_ratio >= 0.8:
img_score += 1
else:
issues.append({'category': 'technical', 'severity': 'medium',
'issue': f'{images["without_alt_text"]} images missing alt text'})
if images['modern_format_count'] > 0:
img_score += 1
else:
img_score = 1 # No images is OK for some content types
img_score = min(img_score, 3)
tech += img_score
tech_breakdown['images'] = img_score
# Structured data (tables, lists): 2 pts
struct_data = analysis['structured_data']
sdata_score = 0
if struct_data['table_count'] >= 1:
sdata_score += 1
if struct_data['unordered_list_items'] + struct_data['ordered_list_items'] >= 3:
sdata_score += 1
sdata_score = min(sdata_score, 2)
tech += sdata_score
tech_breakdown['structured_data'] = sdata_score
# Page speed signals: 2 pts
speed_score = 0
# Check for lazy loading
if re.search(r'loading=["\']lazy["\']', analysis.get('_raw_content', '')):
speed_score += 1
# Check for modern image formats or optimization attributes
if images.get('modern_format_count', 0) > 0:
speed_score += 1
elif images['count'] == 0:
speed_score = 1 # No images to slow things down
speed_score = min(speed_score, 2)
tech += speed_score
tech_breakdown['page_speed'] = speed_score
# Mobile-friendly: 2 pts. Score observable rendering signals, never prose
# length.
mobile_score = 0
raw_content = analysis.get('_raw_content', '')
fixed_width_layout = bool(re.search(
r'style=["\'][^"\']*\bwidth\s*:\s*(?:1\d{3}|[2-9]\d{3,})px',
raw_content,
re.IGNORECASE,
))
if not fixed_width_layout:
mobile_score += 1
if re.search(
r'srcset|<picture|name=["\']viewport["\']',
raw_content,
re.IGNORECASE,
):
mobile_score += 1
elif images['count'] == 0 and not re.search(
r'<(?:iframe|video)\b', raw_content, re.IGNORECASE
):
mobile_score += 1
mobile_score = min(mobile_score, 2)
tech += mobile_score
tech_breakdown['mobile'] = mobile_score
# OG/social meta tags: 2 pts
social = analysis['social_meta']
social_score = 0
if social['og_tags_found'] >= 2:
social_score += 1
if social['has_social_image']:
social_score += 1
elif any(f in fm for f in ('image', 'thumbnail', 'cover')):
social_score += 1
social_score = min(social_score, 2)
tech += social_score
tech_breakdown['social_meta'] = social_score
tech = min(tech, 15)
category_details['technical_elements'] = {'score': tech, 'max': 15, 'breakdown': tech_breakdown}
# ===================================================================
# AI CITATION READINESS (15 pts)
# ===================================================================
ai = 0
ai_breakdown: dict[str, Any] = {}
ai_ready = analysis['ai_citation_readiness']
cit = analysis['citations']
# Evidence-backed, self-contained sections: 4 pts. No fixed word band.
cite_score = 0
supported_sections = ai_ready.get('evidence_backed_sections', 0)
self_contained = ai_ready.get('self_contained_sections', 0)
if supported_sections >= 3 and self_contained >= 2 and cit['unsourced_statistics'] == 0:
cite_score = 4
elif supported_sections >= 2 and self_contained >= 1:
cite_score = 3
elif supported_sections >= 1:
cite_score = 2
elif cit['inline_citations'] >= 1:
cite_score = 1
else:
issues.append({'category': 'ai_citation', 'severity': 'medium',
'issue': 'No evidence-backed sections found - support reusable claims with clear sources or transparent original evidence'})
ai += cite_score
ai_breakdown['citability'] = cite_score
# Purpose fit: 3 pts. Declarative headings and optional FAQs are valid.
purpose_score = 0
if ai_ready.get('purpose_statement'):
purpose_score += 1
if headings['hierarchy_clean'] and headings['h2_count'] >= 1:
purpose_score += 1
if engagement['example_count'] >= 1 or ai_ready.get('has_tldr'):
purpose_score += 1
ai += purpose_score
ai_breakdown['purpose_fit'] = purpose_score
# Entity clarity: 3 pts
ent_score = 0
ed = ai_ready['entity_definitions']
if ed >= 3:
ent_score = 3
elif ed >= 1:
ent_score = 2
else:
ent_score = 0
issues.append({'category': 'ai_citation', 'severity': 'low',
'issue': 'No entity definitions found - use **term** is/are patterns'})
ai += ent_score
ai_breakdown['entity_clarity'] = ent_score
# Reader-useful structure: 3 pts. No single format is mandatory.
ext_score = 0
if ai_ready['has_tldr'] or engagement['example_count'] >= 1:
ext_score += 1
if struct_data['table_count'] >= 1:
ext_score += 1
elif struct_data['unordered_list_items'] + struct_data['ordered_list_items'] >= 3:
ext_score += 1
if ai_ready.get('relevant_media_count', 0) >= 1:
ext_score += 1
ext_score = min(ext_score, 3)
ai += ext_score
ai_breakdown['extraction'] = ext_score
# AI crawler accessibility: 2 pts
crawl_score = 2 # Default: accessible
if ai_ready['has_robots_restriction']:
crawl_score = 0
issues.append({'category': 'ai_citation', 'severity': 'medium',
'issue': 'Robots/noai restriction detected - may block AI crawlers'})
ai += crawl_score
ai_breakdown['crawler_access'] = crawl_score
ai = min(ai, 15)
category_details['ai_citation_readiness'] = {'score': ai, 'max': 15, 'breakdown': ai_breakdown}
# ===================================================================
# TOTAL
# ===================================================================
total = cq + seo + eeat + tech + ai
if total >= 90:
rating = 'Exceptional'
elif total >= 80:
rating = 'Strong'
elif total >= 70:
rating = 'Acceptable'
elif total >= 60:
rating = 'Below Standard'
else:
rating = 'Rewrite'
# Sort issues by severity
severity_order = {'high': 0, 'medium': 1, 'low': 2}
issues.sort(key=lambda x: severity_order.get(x.get('severity', 'low'), 3))
return {
'total': total,
'rating': rating,
'methodology': 'internal_editorial_readiness_heuristic',
'calibrated_probability': False,
'categories': {
'content_quality': cq,
'seo_optimization': seo,
'eeat_signals': eeat,
'technical_elements': tech,
'ai_citation_readiness': ai,
},
'category_details': category_details,
'issues': issues,
'content_type': _detect_content_type(analysis['frontmatter'], analysis['headings'], ''),
}
# ---------------------------------------------------------------------------
# File analysis orchestrator
# ---------------------------------------------------------------------------
def analyze_file(file_path: str) -> dict[str, Any]:
"""Analyze a single blog file with all analyzers."""
path = Path(file_path)
if not path.exists():
return {'error': f'File not found: {file_path}'}
try:
content = _read_safely(path)
except (OSError, UnicodeDecodeError, ValueError) as exc:
return {'error': f'Could not analyze {file_path}: {exc}'}
if path.suffix.lower() == '.html':
frontmatter, body = extract_html_for_analysis(content)
else:
frontmatter = extract_frontmatter(content)
body = strip_frontmatter(content)
language = _detect_language(frontmatter, body)
# Strip markdown formatting for plain-text analysis
plain_text = _plain_text_for_analysis(body)
headings_info = analyze_headings(body)
sentences_info = analyze_sentences(plain_text)
faq_info = analyze_faq(body)
ai_citation_readiness = analyze_ai_citation_readiness(
body, headings_info, faq_info, language
)
if path.suffix.lower() == '.html':
ai_citation_readiness['has_robots_restriction'] = bool(re.search(
r'(?is)<meta[^>]+name=["\']robots["\'][^>]+content=["\'][^"\']*(?:noindex|noai)[^"\']*["\']',
content,
))
analysis: dict[str, Any] = {
'file': str(path),
'format': path.suffix,
'language': language,
'methodology': {
'name': 'internal editorial readiness heuristic',
'calibrated_probability': False,
'authorship_classifier': False,
},
'frontmatter': frontmatter,
'headings': headings_info,
'paragraphs': analyze_paragraphs(body),
'images': analyze_images(content),
'charts': analyze_charts(content),
'citations': analyze_citations(body),
'faq': faq_info,
'freshness': analyze_freshness(frontmatter),
'self_promotion': analyze_self_promotion(body),
'readability': analyze_readability(plain_text, language),
'sentences': sentences_info,
'ai_signals': analyze_ai_signals(plain_text, sentences_info),
'passive_voice': analyze_passive_voice(plain_text),
'transition_words': analyze_transition_words(plain_text),
'ai_trigger_words': analyze_ai_trigger_words(plain_text),
'schema': analyze_schema(content),
'links': analyze_links(body),
'originality': analyze_originality(body, language),
'engagement': analyze_engagement(body),
'ai_citation_readiness': ai_citation_readiness,
'social_meta': analyze_social_meta(content, frontmatter),
'structured_data': analyze_structured_data(body),
# Internal refs used by scoring (not included in output)
'_body_text': body,
'_raw_content': content,
}
analysis['score'] = calculate_score(analysis)
# Remove internal-only keys before returning
analysis.pop('_body_text', None)
analysis.pop('_raw_content', None)
return analysis
# ---------------------------------------------------------------------------
# Output formatters
# ---------------------------------------------------------------------------
def _format_markdown(result: dict[str, Any]) -> str:
"""Format analysis result as a human-readable markdown report."""
if 'error' in result:
return f"## Error\n\n{result['error']}"
score = result['score']
lines: list[str] = []
filename = Path(result['file']).name
lines.append(f"## Blog Quality Report: {filename}")
lines.append('')
lines.append(f"### Overall Score: {score['total']}/100 - {score['rating']}")
lines.append('')
# Category table
lines.append('| Category | Score | Max |')
lines.append('|----------|------:|----:|')
cat_names = {
'content_quality': 'Content Quality',
'seo_optimization': 'SEO Optimization',
'eeat_signals': 'E-E-A-T Signals',
'technical_elements': 'Technical Elements',
'ai_citation_readiness': 'AI Citation Readiness',
}
cat_maxes = {
'content_quality': 30,
'seo_optimization': 25,
'eeat_signals': 15,
'technical_elements': 15,
'ai_citation_readiness': 15,
}
for key, label in cat_names.items():
s = score['categories'].get(key, 0)
m = cat_maxes[key]
lines.append(f'| {label} | {s} | {m} |')
lines.append('')
# Editorial style diagnostics
ai_sig = result.get('ai_signals', {})
lines.append('### Editorial Style Diagnostics')
lines.append('- These descriptive observations do not infer authorship and do not affect the score.')
lines.append(f'- Sentence-length variation: {ai_sig.get("burstiness", 0)}')
lines.append(f'- Configured style phrases: {ai_sig.get("ai_phrase_count", 0)} found')
lines.append(f'- Vocabulary diversity sample: {ai_sig.get("vocabulary_diversity_ttr", 0)}')
lines.append('')
# Readability
read = result.get('readability', {})
passive = result.get('passive_voice', {})
transitions = result.get('transition_words', {})
ai_triggers = result.get('ai_trigger_words', {})
sents = result.get('sentences', {})
lines.append('### Readability')
lines.append(f'- Flesch Reading Ease: {read.get("flesch_reading_ease", "N/A")} (target: 60-70)')
if read.get('flesch_kincaid_grade'):
lines.append(f'- Flesch-Kincaid Grade: {read.get("flesch_kincaid_grade")} (target: 7-8)')
lines.append(f'- Reading time: {read.get("reading_time_minutes", "N/A")} minutes')
lines.append(f'- Passive voice: {passive.get("passive_pct", "N/A")}% (descriptive only)')
lines.append(f'- Transition words: {transitions.get("transition_pct", "N/A")}% (target: 20-30%)')
lines.append(f'- Project style-list terms: {ai_triggers.get("per_1k", "N/A")}/1K (advisory)')
lines.append(f'- Sentences over 20 words: {sents.get("over_20_pct", "N/A")}% (target: ≤25%)')
if ai_triggers.get('found'):
trigger_list = ', '.join(f'{t["word"]}({t["count"]})' for t in ai_triggers['found'][:5])
lines.append(f'- Trigger words found: {trigger_list}')
if read.get('estimated'):
lines.append('- *(Estimated - install textstat for accurate metrics)*')
lines.append('')
# Issues
issues = score.get('issues', [])
if issues:
lines.append('### Issues')
for issue in issues:
sev = issue.get('severity', 'low').upper()
lines.append(f'- [{sev}] {issue["issue"]}')
lines.append('')
else:
lines.append('### Issues')
lines.append('No issues detected.')
lines.append('')
# Content info
lines.append('### Content Info')
lines.append(f'- Word count: {result["paragraphs"]["total_word_count"]}')
lines.append(f'- Content type: {score.get("content_type", "default")}')
lines.append(f'- Sentences: {result["sentences"]["count"]}')
lines.append(f'- Headings: {result["headings"]["total"]}')
lines.append(f'- Internal links: {result["links"]["internal_count"]}')
lines.append(f'- External links: {result["links"]["external_count"]}')
lines.append(f'- Images: {result["images"]["count"]}')
lines.append('')
return '\n'.join(lines)
def _format_table(result: dict[str, Any]) -> str:
"""Format analysis result as a compact table."""
if 'error' in result:
return f"ERROR: {result['error']}"
score = result['score']
filename = Path(result['file']).name
cats = score['categories']
lines: list[str] = []
lines.append(f'{filename} [{score["total"]}/100 {score["rating"]}]')
lines.append(f' Content: {cats["content_quality"]}/30 '
f'SEO: {cats["seo_optimization"]}/25 '
f'E-E-A-T: {cats["eeat_signals"]}/15 '
f'Tech: {cats["technical_elements"]}/15 '
f'AI-Cite: {cats["ai_citation_readiness"]}/15')
issues = score.get('issues', [])
high_issues = [i for i in issues if i.get('severity') == 'high']
if high_issues:
lines.append(f' HIGH: {"; ".join(i["issue"] for i in high_issues[:3])}')
return '\n'.join(lines)
def _format_fix(result: dict[str, Any]) -> str:
"""Output specific, actionable fixes prioritized by impact."""
if 'error' in result:
return f"ERROR: {result['error']}"
score = result['score']
issues = score.get('issues', [])
filename = Path(result['file']).name
lines: list[str] = []
lines.append(f"Fixes for {filename} (Score: {score['total']}/100)")
lines.append('=' * 60)
if not issues:
lines.append('No issues found - content meets all quality checks.')
return '\n'.join(lines)
for i, issue in enumerate(issues, 1):
sev = issue.get('severity', 'low').upper()
cat = issue.get('category', '').replace('_', ' ').title()
lines.append(f'{i}. [{sev}] ({cat}) {issue["issue"]}')
return '\n'.join(lines)
def _format_category_detail(result: dict[str, Any], category: str) -> str:
"""Output detailed breakdown for a single category."""
if 'error' in result:
return f"ERROR: {result['error']}"
score = result['score']
cat_map = {
'content': 'content_quality',
'seo': 'seo_optimization',
'eeat': 'eeat_signals',
'technical': 'technical_elements',
'tech': 'technical_elements',
'ai': 'ai_citation_readiness',
'ai_citation': 'ai_citation_readiness',
'citation': 'ai_citation_readiness',
}
cat_key = cat_map.get(category.lower(), category.lower())
details = score.get('category_details', {}).get(cat_key)
if not details:
available = ', '.join(cat_map.keys())
return f"Unknown category: '{category}'. Available: {available}"
cat_labels = {
'content_quality': 'Content Quality',
'seo_optimization': 'SEO Optimization',
'eeat_signals': 'E-E-A-T Signals',
'technical_elements': 'Technical Elements',
'ai_citation_readiness': 'AI Citation Readiness',
}
lines: list[str] = []
label = cat_labels.get(cat_key, cat_key)
lines.append(f"{label}: {details['score']}/{details['max']}")
lines.append('-' * 40)
breakdown = details.get('breakdown', {})
for sub_key, sub_score in breakdown.items():
lines.append(f" {sub_key.replace('_', ' ').title()}: {sub_score}")
# Category-specific issues
cat_issues = [i for i in score.get('issues', []) if i.get('category') == cat_key or
i.get('category') == category.lower()]
if cat_issues:
lines.append('')
lines.append('Issues:')
for issue in cat_issues:
lines.append(f" - [{issue['severity'].upper()}] {issue['issue']}")
return '\n'.join(lines)
# ---------------------------------------------------------------------------
# Batch processing
# ---------------------------------------------------------------------------
def _process_batch(directory: Path, sort_key: str = 'score') -> dict[str, Any]:
"""Recursively analyze blog files while excluding generated trees."""
results: list[dict[str, Any]] = []
root = directory.resolve()
skip_directories = {
'.git', '.next', '.astro', '.cache', '__pycache__',
'node_modules', 'vendor', 'dist', 'build', 'coverage', 'reports',
'artifacts', 'outputs',
}
candidates: list[Path] = []
for current_root, dirnames, filenames in os.walk(root, followlinks=False):
dirnames[:] = sorted(
name for name in dirnames
if name not in skip_directories
and not name.startswith('.')
and not (Path(current_root) / name).is_symlink()
)
for filename in sorted(filenames):
if Path(filename).suffix.lower() not in {'.md', '.mdx', '.html'}:
continue
path = Path(current_root) / filename
if path.is_symlink() or not path.is_file():
continue
try:
path.resolve().relative_to(root)
except ValueError:
continue
candidates.append(path)
for path in candidates:
# One malformed file must not abort the whole batch; report its error
# in that file's own result and keep going.
try:
results.append(analyze_file(str(path)))
except Exception as exc: # noqa: BLE001 - untrusted input, isolate per file
results.append({'file': str(path), 'error': f'Could not analyze {path}: {type(exc).__name__}: {exc}'})
# Sort
if sort_key == 'score':
results.sort(key=lambda r: (
-r.get('score', {}).get('total', 0),
r.get('file', ''),
))
elif sort_key == 'name':
results.sort(key=lambda r: r.get('file', ''))
elif sort_key == 'words':
results.sort(key=lambda r: (
-r.get('paragraphs', {}).get('total_word_count', 0),
r.get('file', ''),
))
return {'batch': True, 'count': len(results), 'results': results}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main(args: argparse.Namespace) -> None:
"""Main execution function."""
path = Path(args.input)
fmt = getattr(args, 'format', 'json')
category = getattr(args, 'category', None)
fix_mode = getattr(args, 'fix', False)
sort_key = getattr(args, 'sort', 'score')
# Batch mode
if path.is_dir() and getattr(args, 'batch', False):
batch_result = _process_batch(path, sort_key)
if fmt == 'markdown':
for r in batch_result['results']:
print(_format_markdown(r))
print('---\n')
elif fmt == 'table':
for r in batch_result['results']:
print(_format_table(r))
print(f'\nTotal: {batch_result["count"]} files')
else:
output = json.dumps(batch_result, indent=2)
if args.output:
_safe_write_text(args.output, output)
print(f'Report saved to {args.output}', file=sys.stderr)
else:
print(output)
return
# Single file mode
if not path.is_file():
error = {'error': f'Path not found or not a file: {args.input}'}
if fmt == 'json':
print(json.dumps(error, indent=2))
else:
print(f"ERROR: {error['error']}")
sys.exit(1)
result = analyze_file(str(path))
# Category detail mode
if category:
print(_format_category_detail(result, category))
return
# Fix mode
if fix_mode:
print(_format_fix(result))
return
# Format output
if fmt == 'markdown':
output = _format_markdown(result)
if args.output:
_safe_write_text(args.output, output)
print(f'Report saved to {args.output}', file=sys.stderr)
else:
print(output)
elif fmt == 'table':
print(_format_table(result))
else:
output = json.dumps(result, indent=2)
if args.output:
_safe_write_text(args.output, output)
print(f'Report saved to {args.output}', file=sys.stderr)
else:
print(output)
if __name__ == '__main__':
_print_dependency_notice()
parser = argparse.ArgumentParser(
description='Blog Quality Analyzer - 5-category, 100-point scoring system',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
python3 analyze_blog.py post.md Default JSON output
python3 analyze_blog.py post.md --format markdown Markdown report
python3 analyze_blog.py post.md --format table Compact table
python3 analyze_blog.py ./posts --batch --sort score Batch analysis
python3 analyze_blog.py post.md --category seo Single category detail
python3 analyze_blog.py post.md --fix Prioritized fix list
Scoring Categories (100 points):
Content Quality 30 pts Depth, readability, originality, structure
SEO Optimization 25 pts Title, headings, keywords, linking, meta
E-E-A-T Signals 15 pts Author, citations, trust, experience
Technical Elements 15 pts Schema, images, structured data, speed
AI Citation Readiness 15 pts Citability, Q&A, entities, extraction
Rating Bands:
90-100 Exceptional 80-89 Strong 70-79 Acceptable
60-69 Below Standard <60 Rewrite
Optional dependencies (graceful degradation):
pip install textstat beautifulsoup4
""",
)
parser.add_argument('--version', action='version', version=f'%(prog)s {_project_version()}')
parser.add_argument('input', help='Blog file path or directory (with --batch)')
parser.add_argument('--output', '-o', help='Output file path')
parser.add_argument('--format', '-f', choices=['json', 'markdown', 'table'],
default='json', help='Output format (default: json)')
parser.add_argument('--batch', action='store_true',
help='Analyze all .md/.mdx/.html files in directory')
parser.add_argument('--sort', choices=['score', 'name', 'words'],
default='score', help='Sort order for batch mode (default: score)')
parser.add_argument('--category', '-c',
help='Show detailed breakdown for a single category '
'(content, seo, eeat, technical, ai)')
parser.add_argument('--fix', action='store_true',
help='Output prioritized list of specific fixes')
args = parser.parse_args()
try:
main(args)
except Exception as e:
print(json.dumps({'error': str(e)}), file=sys.stderr)
sys.exit(1)