Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(web-analytics): keep site navigation out of autopilot drafts (#122203)

Jordan Mryyanb0f503431733
diff --git a/products/web_analytics/backend/content_autopilot/edits.py b/products/web_analytics/backend/content_autopilot/edits.py
index 4760e051c3be..38f4498f63e9 100644
--- a/products/web_analytics/backend/content_autopilot/edits.py
+++ b/products/web_analytics/backend/content_autopilot/edits.py
@@ -79,6 +79,10 @@ def page_headings(markdown: str) -> list[str]:
return [parsed[1] for parsed in _headings(markdown.splitlines()).values() if parsed[0] >= 2]
+def first_h1(lines: list[str]) -> tuple[int, str] | None:
+ return next(((index, text) for index, (level, text) in _headings(lines).items() if level == 1), None)
+
+
def _section_end(lines: list[str], start: int, level: int) -> int:
return next(
(index for index, parsed in _headings(lines).items() if index > start and parsed[0] <= level),
@@ -127,7 +131,8 @@ def _append(lines: list[str], edit: PageEdit) -> list[str]:
def _replace_intro(lines: list[str], edit: PageEdit) -> list[str]:
headings = _headings(lines)
- h1 = next((index for index, parsed in headings.items() if parsed[0] == 1), -1)
+ heading = first_h1(lines)
+ h1 = heading[0] if heading else -1
new_lines = edit.markdown.splitlines()
replaces_title = bool(new_lines) and (parsed := _heading(new_lines[0])) is not None and parsed[0] == 1
start = h1 if replaces_title and h1 >= 0 else h1 + 1
diff --git a/products/web_analytics/backend/content_autopilot/generation.py b/products/web_analytics/backend/content_autopilot/generation.py
index b47cac2ddfa8..5d3599b79dd6 100644
--- a/products/web_analytics/backend/content_autopilot/generation.py
+++ b/products/web_analytics/backend/content_autopilot/generation.py
@@ -15,7 +15,7 @@ from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from posthog.security.llm_prompt_sanitization import sanitize_user_text, strip_llm_framing_markers
-from products.web_analytics.backend.content_autopilot.edits import PageEdit, apply_edits, page_headings
+from products.web_analytics.backend.content_autopilot.edits import PageEdit, apply_edits, first_h1, page_headings
from products.web_analytics.backend.content_autopilot.llm import (
ContentAutopilotLLMError,
Effort,
@@ -45,7 +45,7 @@ from products.web_analytics.backend.content_autopilot.research import (
gather_research,
search_site_pages,
)
-from products.web_analytics.backend.content_autopilot.site_discovery import site_host
+from products.web_analytics.backend.content_autopilot.site_discovery import is_site_link, site_host
from products.web_analytics.backend.content_autopilot.validation import (
LINK_RE,
JudgeVerdict,
@@ -304,17 +304,20 @@ def _draft_from_payload(payload: dict[str, Any], *, original_markdown: str = "")
)
)
)
+ title = sanitize_user_text(str(payload.get("title", "")), 300)
if "edits" in payload:
applied = apply_edits(original_markdown, list(edits))
markdown = applied.markdown
new_markdown = "\n\n".join(edit.markdown for edit in edits)
unplaced = applied.unplaced
+ if heading := first_h1(markdown.splitlines()):
+ title = sanitize_user_text(heading[1], 300)
else:
markdown = strip_llm_framing_markers(str(payload.get("markdown", "")), 400_000)
new_markdown = markdown
unplaced = ()
return Draft(
- title=sanitize_user_text(str(payload.get("title", "")), 300),
+ title=title,
description=sanitize_user_text(str(payload.get("description", "")), 300),
url_path=sanitize_user_text(str(payload.get("url_path", "")), 500),
markdown=markdown,
@@ -483,10 +486,8 @@ def _internal_link_urls(markdown: str, origin: str) -> list[str]:
host = site_host(origin)
urls: dict[str, None] = {}
for target in LINK_RE.findall(markdown):
- if target.startswith("/") and not target.startswith("//"):
- urls[f"{origin}{target}"] = None
- elif (target_host := site_host(target)) and target_host == host:
- urls[target] = None
+ if is_site_link(target, host):
+ urls[f"{origin}{target}" if target.startswith("/") else target] = None
return list(urls)
diff --git a/products/web_analytics/backend/content_autopilot/prompts.py b/products/web_analytics/backend/content_autopilot/prompts.py
index 49911285374f..17182af89efb 100644
--- a/products/web_analytics/backend/content_autopilot/prompts.py
+++ b/products/web_analytics/backend/content_autopilot/prompts.py
@@ -33,7 +33,7 @@ Work from the evidence you are given:
- Competitor pages show what the engines found useful: coverage, structure, depth. Use them to find gaps, never as a source of facts about this site.
- Engine answers show what the engines currently say. Note what they get wrong or leave out about this site.
-Decide whether the site needs a new page or whether an existing page should be improved. When you are told which page to improve, keep it. Otherwise, if one of the listed site pages already targets the question, recommend improving it and set target_page to its URL. Leave target_page empty for a new page.
+Decide whether the site needs a new page or whether an existing page should be improved. When you are told which page to improve, keep it. Otherwise, if one of the listed site pages is mainly about this question, recommend improving it and set target_page to its URL. A page that only mentions the topic in passing, or that is about a different feature, is not a match: plan a new page instead. Leave target_page empty for a new page.
Plan within the word budget you are given. When you improve a page, plan the smallest set of changes that closes the gap, not a rewrite.
@@ -120,6 +120,7 @@ Length
Structure for answer engines
- Start with one H1. The first paragraph after it answers the question directly in 40 to 70 words, so an engine can quote it on its own.
- If the brief gives a disambiguation, name the product in full with its category in that first paragraph, for example "Plausible Analytics, the privacy-focused web analytics tool".
+- Otherwise name the site's own product plainly. Don't describe it with its category, such as "Acme, the analytics platform,", because readers of the site already know what it is.
- Use H2 headings phrased as the questions people ask. Keep each section self-contained.
- Prefer concrete steps, numbers, and short tables over long prose. Use a comparison table when the question compares options.
- End with a "## Frequently asked questions" section of 3 to 5 questions, each with a short, direct answer. Repeat those pairs in the faq field.
@@ -239,7 +240,7 @@ JUDGE_SYSTEM_PROMPT = f"""You review a draft web page before a person publishes
Check three things and report them honestly. Do not rewrite the draft.
-1. Grounding: check only the text in <draft>. Every factual claim about the company, its product, pricing, features, or limits must be supported by a <site_page>. Every specific factual claim about another company's product (prices, plan limits, features it has or lacks, integrations, hosting, compliance, company news) must be supported by a <competitor_page> from that company or by a <site_page>. Naming a product or saying what kind of product it is (for example "Sentry is an error monitoring tool") does not need a source, and neither does general industry knowledge. List each unsupported claim.
+1. Grounding: check only the text in <draft>. Every factual claim about the company, its product, pricing, features, or limits must be supported by a <site_page>. Every specific factual claim about another company's product (prices, plan limits, features it has or lacks, integrations, hosting, compliance, company news) must be supported by a <competitor_page> from that company or by a <site_page>. Naming a product or saying what kind of product it is (for example "Sentry is an error monitoring tool") does not need a source, and neither does general industry knowledge. List each unsupported claim as the sentence from <draft> that makes it, copied word for word, with no commentary.
2. Answers the prompt: the page directly answers the question in <prompt> early on. When a <page_opening> is given, judge the opening of the finished page from it.
3. Brand rules: the draft follows every brand rule given. List each violation. If no brand rules are given, report that they were followed.
diff --git a/products/web_analytics/backend/content_autopilot/research.py b/products/web_analytics/backend/content_autopilot/research.py
index 595d392f88ae..78fc23677dd0 100644
--- a/products/web_analytics/backend/content_autopilot/research.py
+++ b/products/web_analytics/backend/content_autopilot/research.py
@@ -20,7 +20,12 @@ from posthog.egress.limiter.policies import Priority
from posthog.security.llm_prompt_sanitization import strip_llm_framing_markers
from posthog.security.url_validation import strip_userinfo
-from products.web_analytics.backend.content_autopilot.site_discovery import has_same_public_site, site_host
+from products.web_analytics.backend.content_autopilot.edits import first_h1
+from products.web_analytics.backend.content_autopilot.site_discovery import (
+ has_same_public_site,
+ is_site_link,
+ site_host,
+)
from products.web_analytics.backend.public_url_fetch import PublicUrlFetchError, fetch_public_url
MAX_DOCUMENT_CHARS = 40_000
@@ -31,6 +36,7 @@ MAX_LINK_CANDIDATES = 60
FETCH_SECONDS = 20.0
FIRECRAWL_SOURCE = "content_autopilot"
MAX_SITE_SEARCH_RESULTS = 10
+MAX_STUB_TWIN_WORDS = 80
DocumentOrigin = Literal["site", "competitor"]
@@ -104,6 +110,8 @@ _REPEATED_SPACES_RE = re.compile(r" {2,}")
_CODE_LANGUAGE_RE = re.compile(r"(?:^|\s)lang(?:uage)?-([\w+#.-]+)")
_CODE_BLOCK_TOKEN_RE = re.compile(r"\x00code-block-(\d+)\x00")
_BACKTICK_RUN_RE = re.compile(r"`+")
+_TWIN_METADATA_RE = re.compile(r"^canonical url:", re.IGNORECASE)
+_MARKDOWN_COPY_LINK_RE = re.compile(r"(\]\()([^()\[\]\s?#]+?)\.md(?=[#?)\s])")
def _code_language(attrs: list[tuple[str, str | None]]) -> str:
@@ -255,12 +263,42 @@ def _links_to_page(markdown: str, url: str) -> bool:
return re.search(f"(?:{'|'.join(targets)})/?(?![\\w/.-])", markdown) is not None
-def _without_preamble(markdown: str) -> str:
+def _without_twin_metadata(markdown: str) -> str:
lines = markdown.splitlines()
- h1 = next((index for index, line in enumerate(lines) if line.startswith("# ")), None)
- if h1 is None or not all(not line.strip() or line.startswith(">") for line in lines[:h1]):
+ heading = first_h1(lines)
+ if heading is None:
+ return markdown
+ start = end = heading[0] + 1
+ while end < len(lines) and (
+ not lines[end].strip() or lines[end].startswith(">") or _TWIN_METADATA_RE.match(lines[end])
+ ):
+ end += 1
+ if not any(_TWIN_METADATA_RE.match(line) for line in lines[start:end]):
return markdown
- return "\n".join(lines[h1:])
+ return "\n".join([*lines[:start], "", *lines[end:]])
+
+
+def _with_page_links(markdown: str, url: str) -> str:
+ host = site_host(url)
+
+ def page_link(match: re.Match[str]) -> str:
+ return f"{match.group(1)}{match.group(2)}" if is_site_link(match.group(2), host) else match.group(0)
+
+ return _MARKDOWN_COPY_LINK_RE.sub(page_link, markdown)
+
+
+def _from_first_h1(markdown: str) -> str:
+ lines = markdown.splitlines()
+ heading = first_h1(lines)
+ return "\n".join(lines[heading[0] :]) if heading else markdown
+
+
+def _is_stub_twin(markdown: str, url: str) -> bool:
+ lines = markdown.splitlines()
+ heading = first_h1(lines)
+ if not any(line.strip() for line in lines[heading[0] + 1 if heading else 0 :]):
+ return True
+ return _links_to_page(markdown, url) and len(markdown.split()) <= MAX_STUB_TWIN_WORDS
def _fetch_html_document(url: str, origin: DocumentOrigin) -> SourceDocument | None:
@@ -270,7 +308,7 @@ def _fetch_html_document(url: str, origin: DocumentOrigin) -> SourceDocument | N
extracted = html_to_text(page.body)
if not extracted.text:
return None
- return SourceDocument(url=url, title=extracted.title, text=_clean(extracted.text), origin=origin)
+ return SourceDocument(url=url, title=extracted.title, text=_clean(_from_first_h1(extracted.text)), origin=origin)
def _fetch_rendered_document(url: str, origin: DocumentOrigin) -> SourceDocument | None:
@@ -279,7 +317,9 @@ def _fetch_rendered_document(url: str, origin: DocumentOrigin) -> SourceDocument
if origin == "site" and not (scraped.final_url and has_same_public_site(scraped.final_url, url)):
return _fetch_html_document(url, origin)
if scraped.markdown:
- return SourceDocument(url=url, title=scraped.title or "", text=_clean(scraped.markdown), origin=origin)
+ return SourceDocument(
+ url=url, title=scraped.title or "", text=_clean(_from_first_h1(scraped.markdown)), origin=origin
+ )
except (FirecrawlNotConfigured, FirecrawlScrapeFailed, FirecrawlEgressBudgetExhausted, requests.RequestException):
pass
return _fetch_html_document(url, origin)
@@ -289,10 +329,10 @@ def fetch_site_page(url: str) -> SourceDocument | None:
twin_document: SourceDocument | None = None
twin = _fetch(_markdown_twin(url), "text/markdown,text/plain;q=0.9")
if twin is not None and "html" not in twin.content_type.lower() and twin.body.strip():
- body = _without_preamble(twin.body)
- first_heading = next((line[2:] for line in body.splitlines() if line.startswith("# ")), "")
- twin_document = SourceDocument(url=url, title=first_heading.strip(), text=_clean(body), origin="site")
- if not _links_to_page(twin.body, url):
+ body = _with_page_links(_without_twin_metadata(_from_first_h1(_clean(twin.body))), url)
+ heading = first_h1(body.splitlines())
+ twin_document = SourceDocument(url=url, title=heading[1] if heading else "", text=_clean(body), origin="site")
+ if not _is_stub_twin(body, url):
return twin_document
page_document = _fetch_rendered_document(url, "site")
if page_document is None or (twin_document is not None and len(twin_document.text) >= len(page_document.text)):
diff --git a/products/web_analytics/backend/content_autopilot/site_discovery.py b/products/web_analytics/backend/content_autopilot/site_discovery.py
index 91f95021cf19..0d1110b6f2e2 100644
--- a/products/web_analytics/backend/content_autopilot/site_discovery.py
+++ b/products/web_analytics/backend/content_autopilot/site_discovery.py
@@ -138,6 +138,12 @@ def site_host(url: str) -> str:
return _public_host(url).removeprefix("www.")
+def is_site_link(target: str, host: str) -> bool:
+ if target.startswith("/") and not target.startswith("//"):
+ return True
+ return bool(target_host := site_host(target)) and target_host == host
+
+
def has_same_public_site(first_url: str, second_url: str) -> bool:
first = site_host(first_url)
second = site_host(second_url)
diff --git a/products/web_analytics/backend/content_autopilot/validation.py b/products/web_analytics/backend/content_autopilot/validation.py
index 595fb8e5bdfd..609da056e636 100644
--- a/products/web_analytics/backend/content_autopilot/validation.py
+++ b/products/web_analytics/backend/content_autopilot/validation.py
@@ -5,7 +5,7 @@ from urllib.parse import ParseResult, urlparse
from posthog.dataclasses import frozen
-from products.web_analytics.backend.content_autopilot.edits import heading_levels
+from products.web_analytics.backend.content_autopilot.edits import first_h1, heading_levels
from products.web_analytics.backend.content_autopilot.research import ResearchBundle
from products.web_analytics.backend.content_autopilot.site_discovery import site_host
@@ -96,6 +96,12 @@ class _StructureProblem:
message: str
+def _has_text_before_h1(markdown: str) -> bool:
+ lines = markdown.splitlines()
+ heading = first_h1(lines)
+ return heading is not None and any(line.strip() for line in lines[: heading[0]])
+
+
def _structure_problems(markdown: str) -> dict[str, _StructureProblem]:
levels = heading_levels(markdown)
h1_count = levels.count(1)
@@ -115,6 +121,10 @@ def _structure_problems(markdown: str) -> dict[str, _StructureProblem]:
problems["frontmatter"] = _StructureProblem(
severity=1, message="it starts with frontmatter, which belongs in the content package"
)
+ elif _has_text_before_h1(markdown):
+ problems["preamble"] = _StructureProblem(
+ severity=1, message="it has text before the H1, such as site navigation"
+ )
return problems
@@ -125,7 +135,7 @@ def check_structure(markdown: str, *, baseline: str | None = None) -> Validation
problems = {
key: problem
for key, problem in problems.items()
- if key not in existing or problem.severity > existing[key].severity
+ if key == "preamble" or key not in existing or problem.severity > existing[key].severity
}
return ValidationCheck(
check_key="structure",
diff --git a/products/web_analytics/backend/test/test_content_autopilot_generation.py b/products/web_analytics/backend/test/test_content_autopilot_generation.py
index 216a485e2949..d9a266baf8d4 100644
--- a/products/web_analytics/backend/test/test_content_autopilot_generation.py
+++ b/products/web_analytics/backend/test/test_content_autopilot_generation.py
@@ -252,6 +252,7 @@ class TestContentAutopilotGeneration(BaseTest):
assert proposal.lifecycle_status == ContentAutopilotProposal.LifecycleStatus.READY_FOR_REVIEW
assert proposal.validation_report["passed"] is True
assert proposal.original_markdown == REPLAY_DOC
+ assert proposal.title == "Session replay"
assert proposal.content_package["file_path"] == "docs/session-replay.md"
assert proposal.brief["working_title"] == "How session replay works"
research = ResearchBundle.from_dict(proposal.research)
@@ -771,41 +772,80 @@ class TestSearchSitePages(SimpleTestCase):
class TestFetchSitePage(SimpleTestCase):
@parameterized.expand(
[
- ("full_twin", REPLAY_DOC, REPLAY_DOC, None),
+ ("full_twin", REPLAY_DOC, REPLAY_DOC, ()),
(
"twin_with_a_notice_for_agents",
"> AI agents: this is one page from the docs.\n\n" + REPLAY_DOC,
REPLAY_DOC,
- "AI agents",
+ ("AI agents",),
+ ),
+ (
+ "twin_with_a_canonical_url_and_a_notice_under_the_title",
+ "> AI agents: this is one page from the docs.\n\n# Session replay\n\n"
+ "Canonical URL: https://example.com/docs/session-replay\n\n"
+ "> **Cloud region:** This page uses US Cloud hosts.\n\n"
+ "Session replay records what users do and plays it back with console logs.",
+ "# Session replay\n\nSession replay records what users do and plays it back with console logs.",
+ ("Canonical URL", "Cloud region", "Products Pricing"),
+ ),
+ (
+ "twin_linking_to_other_pages_by_their_markdown_copies",
+ "# Session replay\n\nSee the [dashboard](https://example.com/docs/dashboard.md), "
+ "[heatmaps](/docs/heatmaps.md#setup) and the [README](https://github.com/example/repo/README.md). "
+ "Get the [guide](/download?file=guide.md) or read [setup](/docs/guide#example.md).",
+ "See the [dashboard](https://example.com/docs/dashboard), [heatmaps](/docs/heatmaps#setup) "
+ "and the [README](https://github.com/example/repo/README.md). "
+ "Get the [guide](/download?file=guide.md) or read [setup](/docs/guide#example.md).",
+ ("dashboard.md", "heatmaps.md"),
+ ),
+ (
+ "twin_with_only_a_title_and_metadata",
+ "# Session replay\n\nCanonical URL: https://example.com/docs/session-replay\n\n"
+ "> **Cloud region:** This page uses US Cloud hosts.",
+ "Session replay records every click, scroll and console error in the browser.",
+ ("Canonical URL", "Products Pricing"),
),
(
"stub_twin_linking_to_its_page",
"# Session replay\n\nFull page: https://example.com/docs/session-replay",
"Session replay records every click, scroll and console error in the browser.",
- None,
+ ("Products Pricing", "logo.png"),
),
(
"stub_twin_linking_to_its_page_relatively",
"# Session replay\n\nRead the [full page](/docs/session-replay).",
"Session replay records every click, scroll and console error in the browser.",
- None,
+ ("Products Pricing", "logo.png"),
),
]
)
- def test_prefers_the_richer_page_body(self, _name: str, twin: str, expected_text: str, absent: str | None) -> None:
+ def test_prefers_the_richer_page_body(
+ self, _name: str, twin: str, expected_text: str, absent: tuple[str, ...]
+ ) -> None:
html = "<html><title>Session replay</title><body><p>Session replay records every click, scroll and console error in the browser.</p></body></html>"
+ rendered = FirecrawlScrape(
+ url="https://example.com/docs/session-replay",
+ final_url="https://example.com/docs/session-replay",
+ markdown="Products [Pricing](https://example.com/pricing) DocsCommunity\n\n"
+ + "![](https://cdn.example.com/logo.png)\n\n" * 20
+ + "Copy page\n\n# Session replay\n\n"
+ "Session replay records every click, scroll and console error in the browser.",
+ )
def fetch(url: str, **kwargs: Any) -> FetchedPublicUrl:
if url.endswith(".md"):
return FetchedPublicUrl(status_code=200, headers={"content-type": "text/markdown"}, body=twin.encode())
return FetchedPublicUrl(status_code=200, headers={"content-type": "text/html"}, body=html.encode())
- with patch("products.web_analytics.backend.content_autopilot.research.fetch_public_url", side_effect=fetch):
+ with (
+ patch("products.web_analytics.backend.content_autopilot.research.fetch_public_url", side_effect=fetch),
+ patch("products.web_analytics.backend.content_autopilot.research.scrape", return_value=rendered),
+ ):
document = fetch_site_page("https://example.com/docs/session-replay")
assert document is not None
assert expected_text in document.text
- assert absent is None or absent not in document.text
+ assert [text for text in absent if text in document.text] == []
@parameterized.expand(
[
@@ -891,11 +931,18 @@ class TestFetchSitePage(SimpleTestCase):
class TestValidationChecks(SimpleTestCase):
- def test_unsupported_claims_are_quoted_and_cut_at_a_word(self) -> None:
- claim = (
- '"Inconsistent bucketing: always use the same distinct_id when checking the flag server-side and '
- "client-side. If the ID changes between requests, a user can flip variants."
- )
+ @parameterized.expand(
+ [
+ (
+ "long_claim",
+ '"Inconsistent bucketing: always use the same distinct_id when checking the flag server-side and '
+ "client-side. If the ID changes between requests, a user can flip variants.",
+ '"Inconsistent bucketing: always use the same distinct_id when checking the flag server-side and '
+ 'client-side. If the ID…"',
+ ),
+ ]
+ )
+ def test_unsupported_claims_are_quoted_and_cut_at_a_word(self, _name: str, claim: str, quoted: str) -> None:
verdict = JudgeVerdict(
unsupported_claims=(claim,), answers_prompt=True, answers_prompt_reason="", brand_rule_violations=()
)
@@ -904,10 +951,7 @@ class TestValidationChecks(SimpleTestCase):
check for check in judge_checks(verdict, has_brand_rules=False) if check.check_key == "grounding"
]
- assert grounding.message == (
- "These claims aren't backed by a source: \"Inconsistent bucketing: always use the same distinct_id when "
- 'checking the flag server-side and client-side. If the ID…".'
- )
+ assert grounding.message == f"These claims aren't backed by a source: {quoted}."
@parameterized.expand(
[
@@ -962,12 +1006,18 @@ class TestValidationChecks(SimpleTestCase):
("problem_already_on_the_page", "# A\n\n# B\n\n" + GOOD_MARKDOWN.split("\n", 1)[1], True),
("problem_added_by_the_edit", "# A\n\n" + "word " * 300, False),
("problem_made_worse_by_the_edit", "# A\n\n# B\n\n# C\n\n## Section\n\n" + "word " * 300, False),
+ (
+ "site_navigation_before_the_title_even_when_the_page_had_it",
+ "Products Pricing Docs\n\n![](https://cdn.example.com/logo.png)\n\n# A\n\n# B\n\n## Section\n\n"
+ + "word " * 300,
+ False,
+ ),
]
)
def test_structure_of_an_improvement_ignores_what_the_page_already_had(
self, _name: str, improved: str, passed: bool
) -> None:
- original = "# A\n\n# B\n\n## Section\n\n" + "word " * 300
+ original = "Products Pricing Docs\n\n# A\n\n# B\n\n## Section\n\n" + "word " * 300
assert check_structure(improved, baseline=original).passed is passed