Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

13.0 KB
Raw Blame
#!/usr/bin/env python3
"""Shared head-emission logic for hand-written and generated pages.

``fix_heads.py`` remains the transform for existing HTML.  Article generation
uses the same functions from this module, so canonical URLs, social tags and
JSON-LD have one implementation and one byte layout.
"""

from __future__ import annotations

import html
import json
import re
from typing import Any

import pagedata
import urlmap


BEGIN = "<!-- seo:begin — generated by scripts/seo/fix_heads.py · do not hand-edit -->"
END = "<!-- seo:end -->"

# GA4 measurement tag.  Emitted at the top of every public page's <head> by
# fix_heads.py (pages already on disk) and build_articles.py (generated
# articles), so a new page picks it up the next time either generator runs.
# The inner markup is Google's standard async gtag.js snippet.  The markers
# exist so the transform can delete a previous copy and stay idempotent.
GA4_MEASUREMENT_ID = "G-Z77ETCDJYL"
GA4_BEGIN = "<!-- ga4:begin — generated by scripts/seo/seo_head.py · do not hand-edit -->"
GA4_END = "<!-- ga4:end -->"
RE_GA4 = re.compile(re.escape(GA4_BEGIN) + r".*?" + re.escape(GA4_END) + r"\n?", re.S)

ORG_ID = f"{urlmap.SITE_ORIGIN}/#organization"
SITE_ID = f"{urlmap.SITE_ORIGIN}/#website"
ORG_LOGO = f"{urlmap.SITE_ORIGIN}/brandbook/xerj-mark.svg"
GITHUB = "https://github.com/xerj-org/xerj"
X_PROFILE = "https://x.com/xerj_org"
# One Organization node for xerj.org and hub.xerj.org: same @id, same sameAs
# (tools/corpus-hub-site/gen.py on the corpus-hub branch mirrors these).
ORG_SAME_AS = [GITHUB, X_PROFILE]
# The canonical one-line description, shared with llms.txt, README.md and the
# hub. Change it everywhere at once or not at all.
ORG_DESCRIPTION = ("A new way for AI to search for data: one local Rust binary "
                   "that indexes your code, docs, specs and logs, plus Corpus "
                   "Hub packs, and returns cited answers.")
LICENSE = "https://www.apache.org/licenses/LICENSE-2.0"


def ga4_snippet() -> str:
    """Standard async gtag.js install, wrapped so the head transform can replace it.

    The external script is ``async`` (non-blocking).  The inline block only
    queues the config call; ``'unsafe-inline'`` is already in the site CSP.
    ``landing/_headers`` must allow the tag host and the Analytics collect
    endpoints or the browser drops both.
    """
    mid = GA4_MEASUREMENT_ID
    return (
        f"{GA4_BEGIN}\n"
        "<!-- Google tag (gtag.js) -->\n"
        f'<script async src="https://www.googletagmanager.com/gtag/js?id={mid}"></script>\n'
        "<script>\n"
        "  window.dataLayer = window.dataLayer || [];\n"
        "  function gtag(){dataLayer.push(arguments);}\n"
        "  gtag('js', new Date());\n"
        "\n"
        f"  gtag('config', '{mid}');\n"
        "</script>\n"
        f"{GA4_END}"
    )


def esc(value: str) -> str:
    """Escape an HTML attribute value."""
    return html.escape(value, quote=True)


def text_esc(value: str) -> str:
    """Escape HTML text content."""
    return html.escape(value, quote=False)


def ld(obj: dict[str, Any]) -> str:
    body = json.dumps(obj, ensure_ascii=False, indent=2)
    return f'<script type="application/ld+json">\n{body}\n</script>'


def organization() -> dict[str, Any]:
    return {
        "@context": "https://schema.org",
        "@type": "Organization",
        "@id": ORG_ID,
        "name": "XERJ",
        "alternateName": urlmap.SITE_NAME,
        "url": urlmap.SITE_ORIGIN + "/",
        "logo": ORG_LOGO,
        "description": ORG_DESCRIPTION,
        "sameAs": ORG_SAME_AS,
    }


def website() -> dict[str, Any]:
    return {
        "@context": "https://schema.org",
        "@type": "WebSite",
        "@id": SITE_ID,
        "name": urlmap.SITE_NAME,
        "url": urlmap.SITE_ORIGIN + "/",
        "inLanguage": "en",
        "publisher": {"@id": ORG_ID},
    }


def _team(name: str) -> dict[str, Any]:
    """A named XERJ team (author or reviewer line). Teams are not people, so
    they are typed Organization and tied to the one site Organization node."""
    return {"@type": "Organization", "name": name,
            "parentOrganization": {"@id": ORG_ID}}


def _person_or_org(meta: dict[str, Any]) -> dict[str, Any]:
    name = meta.get("author")
    if name:
        return _team(name)
    return {"@id": ORG_ID}


def _og_image(meta: dict[str, Any]) -> str:
    """The page's og:image — a per-page card when pagedata names one, else
    the one site-wide card generated by mk_og_card.py."""
    path = meta.get("og_image") or urlmap.OG_IMAGE_PATH
    return f"{urlmap.SITE_ORIGIN}/{path}"


def _og_image_alt(meta: dict[str, Any]) -> str:
    return meta.get("og_image_alt") or urlmap.OG_IMAGE_ALT


def page_body(rel: str, meta: dict[str, Any], url: str, title: str,
              desc: str, published: str, modified: str) -> dict[str, Any]:
    kind = meta["kind"]
    typ = pagedata.KIND_TYPE[kind]
    img = _og_image(meta)

    if kind == "software":
        return {
            "@context": "https://schema.org",
            "@type": typ,
            "name": "XERJ",
            "alternateName": urlmap.SITE_NAME,
            "applicationCategory": "DeveloperApplication",
            "applicationSubCategory": "Search engine",
            "operatingSystem": "Linux, macOS, Windows",
            "url": url,
            "description": desc,
            "image": img,
            "license": LICENSE,
            "isAccessibleForFree": True,
            "codeRepository": GITHUB,
            "publisher": {"@id": ORG_ID},
            "mainEntityOfPage": {"@type": "WebPage", "@id": url},
        }

    if kind in pagedata.ARTICLE_KINDS:
        body: dict[str, Any] = {
            "@context": "https://schema.org",
            "@type": typ,
            "headline": title,
            "description": desc,
            "url": url,
            "image": img,
            "inLanguage": "en",
            "datePublished": published,
            "dateModified": modified,
            "author": _person_or_org(meta),
            "publisher": {"@id": ORG_ID},
            "isPartOf": {"@id": SITE_ID},
            "mainEntityOfPage": {"@type": "WebPage", "@id": url},
        }
        reviewer = meta.get("reviewer")
        if reviewer:
            body["reviewedBy"] = _team(reviewer)
        return body

    return {
        "@context": "https://schema.org",
        "@type": typ,
        "name": title,
        "description": desc,
        "url": url,
        "image": img,
        "inLanguage": "en",
        "dateModified": modified,
        "isPartOf": {"@id": SITE_ID},
        "mainEntityOfPage": {"@type": "WebPage", "@id": url},
    }


def breadcrumbs(rel: str, meta: dict[str, Any]) -> dict[str, Any] | None:
    """Home > section hubs > this page.  None for the site root."""
    if rel == "index.html":
        return None

    items: list[dict[str, Any]] = [{
        "@type": "ListItem", "position": 1, "name": "Home",
        "item": urlmap.SITE_ORIGIN + "/",
    }]

    parts = rel.split("/")[:-1]
    if rel.endswith("/index.html"):
        parts = parts[:-1]
    for i in range(len(parts)):
        directory = "/".join(parts[:i + 1])
        hub = pagedata.SECTION_HUBS.get(directory)
        if hub is None:
            continue
        label, path = hub
        items.append({
            "@type": "ListItem", "position": len(items) + 1,
            "name": label, "item": urlmap.SITE_ORIGIN + path,
        })

    items.append({
        "@type": "ListItem", "position": len(items) + 1,
        "name": meta["label"],
    })
    return {
        "@context": "https://schema.org",
        "@type": "BreadcrumbList",
        "itemListElement": items,
    }


_MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)")


def _plain(text: str) -> str:
    """Markdown inline text -> the plain text a reader sees on the page."""
    text = _MD_LINK.sub(r"\1", text)
    return re.sub(r"\s+", " ", text.replace("`", "").replace("**", "")).strip()


def extra_structured_data(url: str, desc: str, meta: dict[str, Any]) -> list[dict[str, Any]]:
    """Opt-in JSON-LD for an article whose source lists `structured_data:`.

    Answers and compare articles opt in to `faq` where every question and
    answer is visible on the page. See README trap 6 and seo_lint rule 46 for the
    policy and the visible-content check.
    """
    kinds = meta.get("structured_data") or ()
    out: list[dict[str, Any]] = []
    if "faq" in kinds and meta.get("faq"):
        out.append({
            "@context": "https://schema.org",
            "@type": "FAQPage",
            "mainEntity": [{
                "@type": "Question",
                "name": _plain(item["q"]),
                "acceptedAnswer": {"@type": "Answer", "text": _plain(item["a"])},
            } for item in meta["faq"]],
        })
    if "howto" in kinds and meta.get("howto_steps"):
        out.append({
            "@context": "https://schema.org",
            "@type": "HowTo",
            "name": _plain(meta.get("h1") or meta["label"]),
            "description": desc,
            "tool": [{"@type": "HowToTool", "name": "xerj (Apache-2.0)"}],
            "step": [{
                "@type": "HowToStep",
                "position": i,
                "name": _plain(step.get("note") or step["cmd"]),
                "text": step["cmd"],
            } for i, step in enumerate(meta["howto_steps"], start=1)],
        })
    if "software" in kinds:
        out.append({
            "@context": "https://schema.org",
            "@type": "SoftwareApplication",
            "name": "XERJ",
            "alternateName": urlmap.SITE_NAME,
            "applicationCategory": "DeveloperApplication",
            "applicationSubCategory": "Search engine",
            "operatingSystem": "Linux, macOS, Windows",
            "url": urlmap.SITE_ORIGIN + "/product",
            "downloadUrl": GITHUB + "/releases/latest",
            "license": LICENSE,
            "isAccessibleForFree": True,
            "offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"},
            "codeRepository": GITHUB,
            "publisher": {"@id": ORG_ID},
        })
    return out


def item_list(url: str, name: str, items: list[dict[str, str]]) -> dict[str, Any]:
    """ItemList of the tools a best-of page compares, in the page's order."""
    return {
        "@context": "https://schema.org",
        "@type": "ItemList",
        "@id": f"{url}#tools",
        "name": _plain(name),
        "numberOfItems": len(items),
        "itemListOrder": "https://schema.org/ItemListUnordered",
        "itemListElement": [{
            "@type": "ListItem",
            "position": i,
            "item": {
                "@type": "SoftwareApplication",
                "name": item["name"],
                "url": item["url"],
                "applicationCategory": "DeveloperApplication",
                "license": item["license"],
                "installUrl": item["url"],
            },
        } for i, item in enumerate(items, start=1)],
    }


def geo_blocks(url: str, meta: dict[str, Any]) -> list[dict[str, Any]]:
    """GEO JSON-LD: an ItemList for a best-of page's compared tools (its
    visible summary table). FAQPage/HowTo stay opt-in in extra_structured_data."""
    if not meta.get("items"):
        return []
    return [item_list(url, meta.get("items_name") or meta.get("title", ""), meta["items"])]


def build_block(rel: str, title: str, desc: str, published: str,
                modified: str, meta: dict[str, Any] | None = None) -> str:
    """Emit the exact SEO block used by ``fix_heads.py``."""
    meta = meta or pagedata.entry(rel)
    url = urlmap.deployed_url(rel)
    img = _og_image(meta)
    og_type = pagedata.KIND_OG_TYPE[meta["kind"]]

    lines = [BEGIN, f'<link rel="canonical" href="{url}">']
    if meta.get("noindex", urlmap.is_noindex(rel)):
        lines.append('<meta name="robots" content="noindex">')

    lines += [
        f'<meta property="og:site_name" content="{esc(urlmap.SITE_NAME)}">',
        f'<meta property="og:type" content="{og_type}">',
        f'<meta property="og:title" content="{esc(title)}">',
        f'<meta property="og:description" content="{esc(desc)}">',
        f'<meta property="og:url" content="{url}">',
        f'<meta property="og:image" content="{img}">',
        f'<meta property="og:image:width" content="{urlmap.OG_IMAGE_WIDTH}">',
        f'<meta property="og:image:height" content="{urlmap.OG_IMAGE_HEIGHT}">',
        f'<meta property="og:image:alt" content="{esc(_og_image_alt(meta))}">',
        '<meta property="og:locale" content="en_US">',
        '<meta name="twitter:card" content="summary_large_image">',
        f'<meta name="twitter:title" content="{esc(title)}">',
        f'<meta name="twitter:description" content="{esc(desc)}">',
        f'<meta name="twitter:image" content="{img}">',
        ld(organization()),
        ld(website()),
        ld(page_body(rel, meta, url, title, desc, published, modified)),
    ]
    lines += [ld(obj) for obj in extra_structured_data(url, desc, meta)]
    lines += [ld(obj) for obj in geo_blocks(url, meta)]
    crumbs = breadcrumbs(rel, meta)
    if crumbs is not None:
        lines.append(ld(crumbs))
    lines.append(END)
    return "\n".join(lines)