Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

8.6 KB
Raw Blame
#!/usr/bin/env python3
"""Idempotent internal-link and site-map-footer pass over landing/*.html.

Two jobs, both safe to re-run:

1. **Rewrite internal `<a href>` values to the URL Cloudflare actually
   serves.**  Every `.html` link on this site costs a 308 round trip for
   users and crawlers, and splits the internal link graph across two
   spellings of the same document.  `/docs/foo.html` -> `/docs/foo`,
   `/docs/index.html` -> `/docs/`, `/index.html` -> `/`.  The mapping lives
   in `urlmap.rewrite_href`, which leaves external URLs, fragments, and every
   asset (`.css`, `.js`, `.json`, `.mp4`, `.png`, `.pdf`, `/get`, `/get.ps1`,
   `openapi.json`, ...) untouched.

   The docs ⌘K search index (`<script type="application/json"
   id="docs-index">`) stores navigation targets in a `slug` field that the
   search UI interpolates straight into `<a href>`, so those are rewritten by
   the same function.

2. **Inject the site-wide site-map footer.**  Fourteen pages — `pricing`,
   `security`, `architecture`, `solutions`, `resources`, the four industry
   verticals and the rest of the enterprise funnel — carried their own nav
   that nothing on the main site linked to.  Google's one stated internal-link
   rule is "every page you care about should have a link from at least one
   other page on your site"; a footer carried by all 77 pages that load
   /style.css satisfies it and puts every section within one click of the
   homepage.

   The block is delimited by HTML comments and regenerated wholesale on each
   run, so re-running produces a zero diff.  Pages in `urlmap.FOOTER_SKIP`
   are skipped because they do not load `/style.css`; they remain footer
   *targets*, which is what the zero-orphan rule actually needs.

    python3 scripts/seo/fix_links.py --write
    python3 scripts/seo/fix_links.py --check
"""

from __future__ import annotations

import argparse
import html
import pathlib
import re
import sys

sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import urlmap  # noqa: E402

FOOTER_BEGIN = ("<!-- seo:footer-begin — generated by scripts/seo/fix_links.py"
                " · do not hand-edit -->")
FOOTER_END = "<!-- seo:footer-end -->"

#: The site map, as (column heading, [(href, anchor text), ...]).
#:
#: Anchor text is the destination's own name in 1-3 words — Google asks for
#: "descriptive, reasonably concise" anchors and explicitly warns off both
#: "click here" and keyword cramming.  Casing is uppercase to match the
#: existing `.footer` / `.nav` typography, which is set in CSS anyway.
FOOTER_MAP: list[tuple[str, list[tuple[str, str]]]] = [
    ("PRODUCT", [
        ("/product", "PRODUCT"),
        ("/architecture/", "ARCHITECTURE"),
        ("/benchmarks/", "BENCHMARKS"),
        ("/pricing/", "PRICING"),
        ("/security/", "SECURITY"),
        ("/playground", "DASHBOARDS"),
        ("/demo/", "REAL-DATA DEMO"),
        ("/aise-demo", "PRODUCT TOUR"),
    ]),
    # The section hubs. Every child page is linked from its hub, so one link
    # per hub here keeps the whole tree two clicks from any page.
    ("LEARN", [
        ("/answers/", "ANSWERS"),
        ("/compare/", "COMPARE"),
        ("/use-cases", "USE CASES"),
        ("/case-studies", "CASE STUDIES"),
        ("/industries/", "INDUSTRIES"),
        ("/blog/", "BLOG"),
    ]),
    ("SOLUTIONS", [
        ("/solutions/", "SOLUTIONS"),
        ("/industries/finserv", "FINANCIAL SERVICES"),
        ("/industries/public-sector", "PUBLIC SECTOR"),
        ("/industries/healthcare", "HEALTHCARE"),
        ("/industries/retail", "RETAIL"),
        # The two flagship long-form use cases that the /use-cases hub's own
        # sub-nav omits — without a hub-level link they hang off one sibling
        # article each, which is the shape orphans grow from.
        ("/use-cases/code-security-audit", "AI CODE-SECURITY AUDIT"),
        ("/use-cases/semantic-analytics", "SEMANTIC ANALYTICS"),
        ("/resources/", "SALES KIT"),
    ]),
    # /brand and /brandbook/ are left out on purpose: they stay reachable by
    # direct URL and link to each other.
    ("DOCS", [
        ("/docs/", "DOCUMENTATION"),
        ("/docs/quickstart", "QUICKSTART"),
        ("/docs/install", "INSTALL"),
        ("/docs/migration-from-es", "MIGRATE FROM ELASTICSEARCH"),
        ("/docs/recipes/", "RECIPES"),
        ("/docs/agents/", "DOCS FOR AI"),
        ("/for-agents", "FOR AI AGENTS"),
        ("/agent-search/", "AGENTIC SEARCH DEMO"),
        ("/llms.txt", "LLMS.TXT"),
        ("https://github.com/xerj-org/xerj", "GITHUB"),
    ]),
]

RE_A_HREF = re.compile(r"(<a\b[^>]*?\shref\s*=\s*)([\"'])(.*?)\2", re.I | re.S)
#: `<script>` bodies are JavaScript, not markup.  The docs search widget builds
#: its result markup with `'<a href="' + e.slug + '">'`; RE_A_HREF matches that
#: as a real anchor, and re-emitting it HTML-escaped turns the JS quotes into
#: `&#x27;` and breaks every result link.  Anchor rewriting therefore skips
#: script bodies entirely — the one thing inside a script that does hold URLs
#: is the `application/json` search index, rewritten by RE_JSON_ISLAND below.
RE_SCRIPT_BLOCK = re.compile(r"<script\b[^>]*>.*?</script\s*>", re.I | re.S)
RE_JSON_ISLAND = re.compile(
    r"(<script\b[^>]*\btype\s*=\s*[\"']application/json[\"'][^>]*>)(.*?)(</script>)",
    re.I | re.S)
RE_SLUG = re.compile(r"(\"slug\"\s*:\s*\")([^\"]+)(\")")
RE_FOOTER_BLOCK = re.compile(
    re.escape(FOOTER_BEGIN) + r".*?" + re.escape(FOOTER_END) + r"\n?", re.S)
RE_FOOTER_TAG = re.compile(r"[ \t]*<footer\b", re.I)
RE_BODY_END = re.compile(r"</body\s*>", re.I)


def render_footer() -> str:
    out = [FOOTER_BEGIN, '<nav class="footer-map" aria-label="Site map">']
    for heading, links in FOOTER_MAP:
        out.append('  <div class="fm-col">')
        out.append(f'    <span class="fm-h">{html.escape(heading)}</span>')
        for href, text in links:
            rel = ' rel="external"' if href.startswith("http") else ""
            out.append(f'    <a href="{html.escape(href, quote=True)}"{rel}>'
                       f"{html.escape(text)}</a>")
        out.append("  </div>")
    out += ["</nav>", FOOTER_END]
    return "\n".join(out)


def rewrite_links(src: str) -> str:
    def sub_a(m: re.Match) -> str:
        head, quote, href = m.group(1), m.group(2), m.group(3)
        new = urlmap.rewrite_href(html.unescape(href))
        return head + quote + html.escape(new, quote=True) + quote

    # Rewrite anchors only outside <script> bodies (see RE_SCRIPT_BLOCK).
    parts, pos = [], 0
    for blk in RE_SCRIPT_BLOCK.finditer(src):
        parts.append(RE_A_HREF.sub(sub_a, src[pos:blk.start()]))
        parts.append(blk.group(0))
        pos = blk.end()
    parts.append(RE_A_HREF.sub(sub_a, src[pos:]))
    out = "".join(parts)

    def sub_island(m: re.Match) -> str:
        body = RE_SLUG.sub(
            lambda s: s.group(1) + urlmap.rewrite_href(s.group(2)) + s.group(3),
            m.group(2))
        return m.group(1) + body + m.group(3)

    return RE_JSON_ISLAND.sub(sub_island, out)


def inject_footer(rel: str, src: str) -> str:
    src = RE_FOOTER_BLOCK.sub("", src)
    if rel in urlmap.FOOTER_SKIP:
        return src

    block = render_footer() + "\n"
    m = RE_FOOTER_TAG.search(src)
    if m:
        return src[:m.start()] + block + src[m.start():]
    m = RE_BODY_END.search(src)
    if m:
        return src[:m.start()] + block + src[m.start():]
    return src.rstrip("\n") + "\n" + block


def transform(rel: str, src: str) -> str:
    return inject_footer(rel, rewrite_links(src))


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--write", action="store_true")
    ap.add_argument("--check", action="store_true")
    ap.add_argument("--root", default=None)
    args = ap.parse_args(argv)
    if not (args.write or args.check):
        ap.print_help()
        return 2

    root = pathlib.Path(args.root) if args.root else urlmap.landing_dir()
    changed: list[str] = []
    for rel in urlmap.iter_pages(root):
        path = root / rel
        src = path.read_text(encoding="utf-8")
        out = transform(rel, src)
        if out != src:
            changed.append(rel)
            if args.write:
                path.write_text(out, encoding="utf-8")

    if args.check:
        if changed:
            print(f"FAIL {len(changed)} page(s) have stale links or footer:",
                  file=sys.stderr)
            for rel in changed[:20]:
                print(f"  {rel}", file=sys.stderr)
            return 1
        print("ok   internal links and site-map footer are current")
        return 0

    print(f"rewrote {len(changed)} of {len(urlmap.iter_pages(root))} pages")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())