AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
<!DOCTYPE html>
<html lang="en" data-theme="day">
<head>
<!-- ga4:begin — generated by scripts/seo/seo_head.py · do not hand-edit -->
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-Z77ETCDJYL"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-Z77ETCDJYL');
</script>
<!-- ga4:end -->
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>XERJ.ai · Docs · Corpus Hub</title>
<meta name="description" content="Ask a pinned, signed, pre-indexed corpus with xerj ask: pulling packs, the node that holds them, offline use, the MCP hub tools, exit codes, and how to contribute a corpus.">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Big+Shoulders+Display:wght@400;700;800;900&family=Inter:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;700&display=swap" rel="stylesheet">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' fill='%230b0b0d'/><text x='50' y='68' font-size='60' font-family='Big Shoulders Display,sans-serif' font-weight='900' fill='%23ffc400' text-anchor='middle'>X</text></svg>">
<script src="/theme.js"></script>
<link rel="stylesheet" href="/style.css?v=90f643e">
<!-- seo:begin — generated by scripts/seo/fix_heads.py · do not hand-edit -->
<link rel="canonical" href="https://xerj.org/docs/corpus-hub">
<meta property="og:site_name" content="XERJ.ai">
<meta property="og:type" content="article">
<meta property="og:title" content="XERJ.ai · Docs · Corpus Hub">
<meta property="og:description" content="Ask a pinned, signed, pre-indexed corpus with xerj ask: pulling packs, the node that holds them, offline use, the MCP hub tools, exit codes, and how to contribute a corpus.">
<meta property="og:url" content="https://xerj.org/docs/corpus-hub">
<meta property="og:image" content="https://xerj.org/og/xerj-card.png">
<meta property="og:image:width" content="1200">
<meta property="og:image:height" content="630">
<meta property="og:image:alt" content="XERJ.ai — the search engine built for AI agents">
<meta property="og:locale" content="en_US">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:title" content="XERJ.ai · Docs · Corpus Hub">
<meta name="twitter:description" content="Ask a pinned, signed, pre-indexed corpus with xerj ask: pulling packs, the node that holds them, offline use, the MCP hub tools, exit codes, and how to contribute a corpus.">
<meta name="twitter:image" content="https://xerj.org/og/xerj-card.png">
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Organization",
"@id": "https://xerj.org/#organization",
"name": "XERJ",
"alternateName": "XERJ.ai",
"url": "https://xerj.org/",
"logo": "https://xerj.org/brandbook/xerj-mark.svg",
"description": "A new way for AI to search for data: one local Rust binary that indexes your code, docs, specs and logs, plus Corpus Hub packs, and returns cited answers.",
"sameAs": [
"https://github.com/xerj-org/xerj",
"https://x.com/xerj_org"
]
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "WebSite",
"@id": "https://xerj.org/#website",
"name": "XERJ.ai",
"url": "https://xerj.org/",
"inLanguage": "en",
"publisher": {
"@id": "https://xerj.org/#organization"
}
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "TechArticle",
"headline": "XERJ.ai · Docs · Corpus Hub",
"description": "Ask a pinned, signed, pre-indexed corpus with xerj ask: pulling packs, the node that holds them, offline use, the MCP hub tools, exit codes, and how to contribute a corpus.",
"url": "https://xerj.org/docs/corpus-hub",
"image": "https://xerj.org/og/xerj-card.png",
"inLanguage": "en",
"datePublished": "2026-10-01",
"dateModified": "2026-10-17",
"author": {
"@id": "https://xerj.org/#organization"
},
"publisher": {
"@id": "https://xerj.org/#organization"
},
"isPartOf": {
"@id": "https://xerj.org/#website"
},
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://xerj.org/docs/corpus-hub"
}
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "BreadcrumbList",
"itemListElement": [
{
"@type": "ListItem",
"position": 1,
"name": "Home",
"item": "https://xerj.org/"
},
{
"@type": "ListItem",
"position": 2,
"name": "Docs",
"item": "https://xerj.org/docs/"
},
{
"@type": "ListItem",
"position": 3,
"name": "Corpus Hub"
}
]
}
</script>
<!-- seo:end -->
</head>
<body>
<nav class="nav" aria-label="Primary">
<a href="/" class="brand">XERJ</a>
<a href="/product">PRODUCT</a>
<a href="/use-cases">USE CASES</a>
<a href="/answers/">ANSWERS</a>
<a href="/benchmarks/">BENCHMARKS</a>
<a href="/blog/">BLOG</a>
<span class="spacer"></span>
<a href="/for-agents">FOR AGENTS</a>
<a href="/docs/" class="active">DOCS</a>
<a href="/playground">DASHBOARDS</a>
<a href="https://hub.xerj.org" rel="external">HUB</a>
<a class="nav-stars" href="https://github.com/xerj-org/xerj" rel="external" title="XERJ on GitHub" aria-label="XERJ on GitHub, 14,225 stars"><svg viewBox="0 0 16 16" width="14" height="14" aria-hidden="true"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"/></svg><span class="nav-stars-n" data-gh-stars="14225">★ 14.2k</span></a>
<a href="https://x.com/xerj_org" rel="external" title="XERJ on X" style="display:inline-flex;align-items:center"><svg width="15" height="15" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M18.244 2.25h3.308l-7.227 8.26 8.502 11.24H16.17l-5.214-6.817L4.99 21.75H1.68l7.73-8.835L1.254 2.25H8.08l4.713 6.231zm-1.161 17.52h1.833L7.084 4.126H5.117z"/></svg></a>
<a href="https://discord.com/invite/yetXFusXfs" rel="external" title="XERJ on Discord" style="display:inline-flex;align-items:center"><svg width="15" height="15" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M20.317 4.37a19.79 19.79 0 00-4.885-1.515.074.074 0 00-.079.037c-.21.375-.444.864-.608 1.25a18.27 18.27 0 00-5.487 0 12.64 12.64 0 00-.617-1.25.077.077 0 00-.079-.037A19.74 19.74 0 003.677 4.37a.07.07 0 00-.032.027C.533 9.046-.32 13.58.099 18.057a.082.082 0 00.031.057 19.9 19.9 0 005.993 3.03.078.078 0 00.084-.028c.462-.63.874-1.295 1.226-1.994a.076.076 0 00-.041-.106 13.1 13.1 0 01-1.872-.892.077.077 0 01-.008-.128 10.2 10.2 0 00.372-.292.074.074 0 01.077-.01c3.928 1.793 8.18 1.793 12.062 0a.074.074 0 01.078.01c.12.098.246.198.373.292a.077.077 0 01-.006.127 12.3 12.3 0 01-1.873.892.077.077 0 00-.041.107c.36.698.772 1.362 1.225 1.993a.076.076 0 00.084.028 19.84 19.84 0 006.002-3.03.077.077 0 00.032-.054c.5-5.177-.838-9.674-3.549-13.66a.061.061 0 00-.031-.03zM8.02 15.33c-1.183 0-2.157-1.085-2.157-2.419 0-1.333.956-2.419 2.157-2.419 1.21 0 2.176 1.096 2.157 2.42 0 1.333-.956 2.418-2.157 2.418zm7.975 0c-1.183 0-2.157-1.085-2.157-2.419 0-1.333.955-2.419 2.157-2.419 1.21 0 2.176 1.096 2.157 2.42 0 1.333-.946 2.418-2.157 2.418z"/></svg></a>
<span class="theme">
<button type="button" data-theme="day" class="active">DAY</button>
<span class="dash">·</span>
<button type="button" data-theme="night">NIGHT</button>
</span>
</nav>
<div class="docs-search" id="docsSearch">
<input type="text" id="docsSearchInput" placeholder="Search docs · keys, flags, endpoints, playbooks…" aria-label="Search docs" autocomplete="off" spellcheck="false">
<span class="hint">⌘ K</span>
<div class="results" id="docsSearchResults" aria-live="polite"></div>
</div>
<div class="docs-shell">
<nav class="docs-nav" aria-label="Docs"> <div class="group">
<span class="label">START</span>
<a href="/docs/" class=""><span class="eyebrow">01</span>Overview</a>
<a href="/docs/quickstart" class=""><span class="eyebrow">02</span>Quickstart</a>
<a href="/docs/install" class=""><span class="eyebrow">03</span>Install</a>
<a href="/docs/migration-from-es" class=""><span class="eyebrow">04</span>Migrate from ES</a>
</div>
<div class="group">
<span class="label">REFERENCE</span>
<a href="/docs/cli" class=""><span class="eyebrow">01</span>CLI</a>
<a href="/docs/config" class=""><span class="eyebrow">02</span>Config TOML</a>
<a href="/docs/env" class=""><span class="eyebrow">03</span>Environment</a>
<a href="/docs/api-native" class=""><span class="eyebrow">04</span>Native REST API</a>
<a href="/docs/api-es-compat" class=""><span class="eyebrow">05</span>ES-compat API</a>
</div>
<div class="group">
<span class="label">DATA MODEL</span>
<a href="/docs/queries" class=""><span class="eyebrow">01</span>Query types</a>
<a href="/docs/aggregations" class=""><span class="eyebrow">02</span>Aggregations</a>
<a href="/docs/analyzers" class=""><span class="eyebrow">03</span>Analyzers</a>
<a href="/docs/vectors" class=""><span class="eyebrow">04</span>Vectors & kNN</a>
</div>
<div class="group">
<span class="label">ENGINE</span>
<a href="/docs/ingest" class=""><span class="eyebrow">01</span>Ingest pipelines</a>
<a href="/docs/storage" class=""><span class="eyebrow">02</span>Storage & WAL</a>
<a href="/docs/compression" class=""><span class="eyebrow">03</span>Compression</a>
<a href="/docs/clustering" class=""><span class="eyebrow">04</span>Clustering</a>
<a href="/docs/metrics" class=""><span class="eyebrow">05</span>Metrics</a>
<a href="/docs/security" class=""><span class="eyebrow">06</span>Auth & TLS</a>
</div>
<div class="group">
<span class="label">OPERATIONS</span>
<a href="/docs/operations" class=""><span class="eyebrow">01</span>Running in production</a>
<a href="/docs/backup-restore" class=""><span class="eyebrow">02</span>Backup & restore</a>
<a href="/docs/upgrades" class=""><span class="eyebrow">03</span>Upgrades</a>
<a href="/docs/troubleshooting" class=""><span class="eyebrow">04</span>Troubleshooting</a>
</div>
<div class="group">
<span class="label">PLAYBOOKS</span>
<a href="/docs/playbooks/siem" class=""><span class="eyebrow">01</span>SIEM</a>
<a href="/docs/playbooks/log-analytics" class=""><span class="eyebrow">02</span>Log analytics</a>
<a href="/docs/playbooks/vector-search" class=""><span class="eyebrow">03</span>Vector · RAG</a>
<a href="/docs/playbooks/full-text" class=""><span class="eyebrow">04</span>Full-text search</a>
<a href="/docs/playbooks/observability" class=""><span class="eyebrow">05</span>Observability</a>
</div>
<div class="group">
<span class="label">RECIPES</span>
<a href="/docs/recipes/" class=""><span class="eyebrow">00</span>All recipes</a>
<a href="/docs/recipes/semantic-search-rag" class=""><span class="eyebrow">01</span>Semantic search & RAG</a>
<a href="/docs/recipes/agentic-memory" class=""><span class="eyebrow">02</span>Agent memory</a>
<a href="/docs/recipes/passage-retrieval" class=""><span class="eyebrow">03</span>Passage retrieval</a>
<a href="/docs/recipes/vector-search-knn" class=""><span class="eyebrow">04</span>Vector search · kNN</a>
<a href="/docs/recipes/vector-quantization" class=""><span class="eyebrow">05</span>Vector quantization</a>
<a href="/docs/recipes/hybrid-search" class=""><span class="eyebrow">06</span>Hybrid search</a>
<a href="/docs/recipes/log-analytics" class=""><span class="eyebrow">07</span>Log analytics</a>
<a href="/docs/recipes/anomaly-detection" class=""><span class="eyebrow">08</span>Anomaly detection</a>
<a href="/docs/recipes/continuous-anomaly-datafeeds" class=""><span class="eyebrow">09</span>Anomaly datafeeds</a>
<a href="/docs/recipes/migrate-from-elasticsearch" class=""><span class="eyebrow">10</span>Migrate from ES</a>
</div>
<div class="group">
<span class="label">CORPUS HUB</span>
<a href="/docs/corpus-hub" class="active"><span class="eyebrow">01</span>Ask, pull & contribute</a>
</div>
</nav>
<main class="docs-content">
<!-- Print-only branded header (hidden on screen, visible in PDF) -->
<div class="print-header" style="display:none;">XERJ</div>
<div class="breadcrumb"><a href="/docs/">DOCS</a><span class="sep">·</span>CORPUS HUB<span class="sep">·</span>ASK, PULL & CONTRIBUTE
<button class="pdf-btn" onclick="window.print()" title="Save this page as PDF">
<svg viewBox="0 0 24 24"><path d="M12 3v12M12 15l-4-4M12 15l4-4"/><path d="M4 17v2a2 2 0 002 2h12a2 2 0 002-2v-2"/></svg>
PDF
</button>
</div>
<div class="kicker"><span>01 · CORPUS HUB</span></div>
<h1>Corpus Hub: ask a pinned corpus</h1>
<p>The Corpus Hub is a catalogue of reference corpora (other projects' source code, standards,
advisory feeds), each pinned to one commit and licence-reviewed by a person. CI builds every
corpus into a <strong>pack</strong>: an index that is already built, signed with the hub key. You
ask a question with one command. XERJ downloads the pack the first time, checks its sha256 and its
ed25519 signature before it extracts anything, restores it into a local node, and answers. Your
machine never clones or indexes the corpus.</p>
<pre class="code">$ xerj ask --def LZ4_VERSION_RELEASE
lz4 pin 2026-10-02 (15 days old) mixed: BSD-2 (lib) / GPL-2.0+ (rest) lz4/lib/lz4.h:133
#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
value: 0
approach-only licence (mixed: BSD-2 (lib) / GPL-2.0+ (rest)): read, do not copy
answered from: local pack
warning: approach-only licence (mixed: BSD-2 (lib) / GPL-2.0+ (rest)): read, do not copy</pre>
<p>Every answer names the corpus, the pinned revision and its age, the licence, and the
<code class="inline">file:line</code> it rests on. Retrieval is lexical (BM25 over the words and
identifiers in the source), not semantic: ask with the names the source uses.</p>
<p><strong>Which binary this page describes.</strong> This page applies to the released
<code class="inline">xerj v1.0.0-rc.97</code> and later. That release has
<code class="inline">xerj ask</code>, <code class="inline">xerj hub</code> and the built-in hub key,
and on 2026-10-21 its <code class="inline">xerj hub sync</code> fetched and verified the production
signed catalogue (324 corpora). The earlier v1.0.0-rc.96 release has
<code class="inline">xerj ask</code> and <code class="inline">xerj hub</code> but no hub key: there,
every pull exits 5 and downloads nothing. To check your binary, run
<code class="inline">xerj hub --help | grep -c build-catalogue</code>; it prints
<code class="inline">1</code> on a binary this page applies to and <code class="inline">0</code> on
v1.0.0-rc.96. A binary that prints 1 but has no key still exits 5 on the first pull, with the
message under <a href="#troubleshooting">When it does not work</a>. The example outputs were
captured from <code class="inline">main</code> at <code class="inline">b3753ddb</code> against a
rehearsal hub (a catalogue of lz4, snappy and zstd signed with a test key), before the production
catalogue was served. Pack sizes, ages and process IDs on your machine will differ; the formats
will not.</p>
<p class="source-note">Source · engine/crates/xerj-hub/src/ (cli.rs, ask.rs, sync.rs, node.rs) · spec docs/CORPUS_HUB_V2_IMPLEMENTATION.md · browse: hub.xerj.org · design write-up: <a href="/blog/corpus-hub-v2">Corpus Hub v2</a></p>
<h2 id="choose">Hub, local, or neither</h2>
<p>Pick the source before you search:</p>
<pre class="code">HUB public material you cannot quote exactly: a library's source at a
pinned commit, a standard, a regulation, an advisory feed.
xerj ask "<question>"
LOCAL files only this machine has: your project, private documents, exports.
xerj autoindex <folder> --prefix <name> (once)
xerj search "<text>" --prefix <name> (pass the same --prefix)
Without --prefix, search and def read the default `ax` prefix.
NEITHER a fact you can already state exactly. Retrieval adds cost there.</pre>
<h2 id="limits">What this build does, and what it does not</h2>
<ul>
<li><strong>Your question stays on this machine.</strong> This build has no hosted-answer client.
It downloads two things, the signed catalogue and the packs you use, and sends no question text,
no document text and no credential to the hub. <code class="inline">--hosted on</code> prints
<code class="inline">hosted answers were requested but this build has no hosted client; answering from a local pack</code>
and answers locally.</li>
<li><strong>Auto-pull cap: 50 MiB per pack</strong> (<code class="inline">AUTO_PULL_MAX_BYTES</code>,
52,428,800 bytes). A question does not auto-download a larger pack; an explicit
<code class="inline">xerj hub pull <slug></code> does fetch it, bounded by the 4 GiB disk budget
(<code class="inline">DISK_BUDGET_BYTES</code>, the largest pack the cache can hold).</li>
<li><strong>Licences.</strong> A corpus with the licence class <code class="inline">excluded</code>
is never pulled. An <code class="inline">approach-only</code> corpus is pulled, and every hit
carries <code class="inline">read, do not copy</code>.</li>
<li><strong>Disk.</strong> The pack cache is pruned to 4.0 GB, least recently used first.</li>
<li><strong>No lockfile yet.</strong> The spec's <code class="inline">xerj.lock</code> is not
written by this build, and <code class="inline">--locked</code>, <code class="inline">xerj hub update</code>
and <code class="inline">xerj hub rm</code> do not exist. To get the same answer on two
machines, compare the pin in the answer's header line (or <code class="inline">pin_sha</code> in
<code class="inline">--json</code>).</li>
<li><strong>MCP tools never download.</strong> The three hub tools answer from packs already on
the machine and from the catalogue already cached; see <a href="#mcp">MCP</a>.</li>
</ul>
<h2 id="first-question">Ask your first question</h2>
<p>You need the <code class="inline">xerj</code> binary and network access to
<code class="inline">hub.xerj.org</code> and to GitHub release downloads. You do not need to start
a node: if nothing answers on <code class="inline">localhost:9200</code>, XERJ starts one of its own
(see <a href="#node">Which node holds the packs</a>).</p>
<p><strong>1. Fetch the catalogue.</strong></p>
<pre class="code">$ xerj hub sync
hub sync: catalogue refreshed (generated 2026-10-17T23:35:36Z, 3 corpora)
hub sync: no cached packs to update
hub sync: 0 updated, 0 current, 0 kept, 0 failed, 0 pruned</pre>
<p>Verify: <code class="inline">echo $?</code> prints <code class="inline">0</code>, and
<code class="inline">xerj hub status</code> ends with <code class="inline">catalogue: 0 days old</code>.
<code class="inline">xerj ask</code> also fetches the catalogue when none is cached, so this step is
optional in a shell. It is required before the MCP tools can answer.</p>
<p><strong>2. Ask for a definition.</strong> <code class="inline">--def</code> is for a value, a
signature or a default. It prints the declaration and, for a constant or a macro, a
<code class="inline">value:</code> line, and no passages. The first answer from a corpus pulls its
pack:</p>
<pre class="code">$ xerj ask --def ZSTD_CLEVEL_DEFAULT
xerj hub: pulling 'zstd' (22.1 MB) — one-time download, later questions answer locally
xerj hub: pulled zstd 01b7154f 23137348 → 5 indices, 7874 definitions
zstd pin 2026-10-02 (15 days old) mixed: BSD-3 (lib) / GPL-2.0 (cli) zstd/lib/zstd.h:134
# define ZSTD_CLEVEL_DEFAULT 3
value: 3
approach-only licence (mixed: BSD-3 (lib) / GPL-2.0 (cli)): read, do not copy
answered from: local pack
warning: approach-only licence (mixed: BSD-3 (lib) / GPL-2.0 (cli)): read, do not copy</pre>
<p>Verify: the output contains <code class="inline">answered from: local pack</code> and
<code class="inline">echo $?</code> prints <code class="inline">0</code>. The two
<code class="inline">xerj hub:</code> lines go to stderr and appear only on the run that downloads.
<code class="inline">xerj ask "what is the value of ZSTD_CLEVEL_DEFAULT"</code> and
<code class="inline">xerj ask ZSTD_CLEVEL_DEFAULT</code> (a lone identifier) ask the same
question.</p>
<p><strong>3. Ask how something works.</strong> Name the corpus with
<code class="inline">--corpus</code> when the question is in plain words (see
<a href="#routing">How a corpus is chosen</a>):</p>
<pre class="code">$ xerj ask --corpus lz4 --max-tokens 120 "frame decoder"
lz4 pin 2026-10-02 (15 days old) mixed: BSD-2 (lib) / GPL-2.0+ (rest) lz4/lib/lz4frame.c:1644
size_t LZ4F_decompress(LZ4F_dctx* dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
…
approach-only licence (mixed: BSD-2 (lib) / GPL-2.0+ (rest)): read, do not copy
answered from: local pack
warning: approach-only licence (mixed: BSD-2 (lib) / GPL-2.0+ (rest)): read, do not copy</pre>
<p>Verify: the first line starts with <code class="inline">lz4 pin</code> and ends with a
<code class="inline">file:line</code>. Without <code class="inline">--max-tokens</code> you get up
to five passages. <code class="inline">--files</code> lists the matching files instead of
passages.</p>
<p><strong>4. See what is on disk.</strong></p>
<pre class="code">$ xerj hub status
pack: lz4 0774d055 15 days old 8.1 MB last used today
pack: zstd 01b7154f 15 days old 26.5 MB last used today
node: running pid 2789576 port 9300
hosted: auto
last sync: today
catalogue: 0 days old</pre>
<p>Verify: one <code class="inline">pack:</code> line for each corpus you asked.</p>
<p><strong>5. Stop the node XERJ started</strong> when you are done:</p>
<pre class="code">$ xerj hub stop
stopped the hub node (pid 2789576)</pre>
<p>Verify: a second <code class="inline">xerj hub stop</code> prints
<code class="inline">no hub node was started by xerj; nothing to stop</code>. It stops only the
process <code class="inline">xerj ask</code> recorded. Never <code class="inline">pkill xerj</code>
or <code class="inline">killall xerj</code>: other XERJ nodes may be running on the machine. The
packs stay on disk; the next <code class="inline">xerj ask</code> starts the node again and
answers without a download.</p>
<h2 id="reading">Reading an answer</h2>
<pre class="code"><corpus> pin <date> (<N> days old) <licence> <file>:<line> one header per hit
<the declaration, or the passage>
value: <literal> def mode, constants and macros
<per-hit licence warning>
answered from: local pack
warning: <…> zero or more</pre>
<p><code class="inline">pin</code> is the source revision the pack was built from. Check its age
when the question is about the current version of a project. Tell the reader which corpus, pin
date and licence the answer rests on. <code class="inline">warning: approach-only licence</code>
means read the source and write your own code; do not copy it.</p>
<p><code class="inline">--json</code> prints the same answer as one object:</p>
<pre class="code">$ xerj ask --json --def LZ4_VERSION_RELEASE
{"corpus":"lz4","pin_sha":"0774d05537f9762f838f7ab541b7765f1a729cb5","pin_date":"2026-10-02",
"age_days":15,"licence_spdx":"mixed: BSD-2 (lib) / GPL-2.0+ (rest)","licence_class":"approach-only",
"answered_from":"local_pack","mode":"def","hits":[{"file":"lz4/lib/lz4.h","line":133,"kind":"const",
"text":"#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */","value":"0",
"score":14.069364,"licence_warning":"approach-only licence (mixed: BSD-2 (lib) / GPL-2.0+ (rest)): read, do not copy"}],
"truncated":false,"warnings":["approach-only licence (mixed: BSD-2 (lib) / GPL-2.0+ (rest)): read, do not copy"]}</pre>
<p>(Wrapped here for width; the binary prints one line.) Verify:
<code class="inline">xerj ask --json --def LZ4_VERSION_RELEASE | jq -r .hits[0].value</code>
prints <code class="inline">0</code>.</p>
<h2 id="routing">How a corpus is chosen</h2>
<p><code class="inline">--corpus <slug></code> skips routing. Without it, XERJ scores each
corpus in the cached catalogue from the dependencies of the project you are in
(<code class="inline">Cargo.lock</code>, <code class="inline">package.json</code>,
<code class="inline">go.mod</code>, <code class="inline">requirements.txt</code>,
<code class="inline">pyproject.toml</code>), from the catalogue's per-corpus term sketch, and from
the exact symbol a definition question names. Three outcomes:</p>
<pre class="code"># one corpus fits: it answers (exit 0)
# more than one fits: the candidates print, one per line (exit 4)
$ xerj ask XXH64
more than one corpus matches; re-run with --corpus <slug>
lz4 — an agent needing fast block compression queries this for the block format and decoder loop
zstd — an agent tuning compression queries this for frame/block formats, window and level internals
# nothing scores high enough (exit 1)
$ xerj "how does the lz4 frame decoder work"
no hub corpus matches; see xerj.org/llms.txt
closest corpora: lz4, snappy, zstd; re-run with --corpus <slug> to ask one of them</pre>
<p>A question in plain words often does not route by itself, even when it contains the
project's name, as the last example shows. When you see <code class="inline">closest corpora:</code>,
run the question again with <code class="inline">--corpus</code>. When there is no
<code class="inline">closest corpora:</code> line, the hub does not hold the material: say so,
and do not answer from memory.</p>
<p><code class="inline">xerj "<question>"</code> is the same command as
<code class="inline">xerj ask "<question>"</code> when the question contains a space and its
first word is not a subcommand. For a single word, write <code class="inline">xerj ask <word></code>.</p>
<h2 id="node">Which node holds the packs</h2>
<p>A pack is restored into a running XERJ node. <code class="inline">xerj ask</code> uses the
first of <code class="inline">--url</code>, <code class="inline">XERJ_URL</code> and
<code class="inline">http://localhost:9200</code>. If that node answers, it is used as it is. If it
does not answer and is a loopback address, XERJ starts its own node on the first free port from
9300, with its data in <code class="inline">~/.xerj/hub/node</code>, and records its PID:</p>
<pre class="code">xerj hub: no node answers at http://127.0.0.1:29599; starting a local node on port 9300 (data: /home/you/.xerj/hub/node, log: /home/you/.xerj/hub/node.log)
xerj hub: node ready on port 9300 (pid 2789576)</pre>
<p>A non-loopback <code class="inline">--url</code> that does not answer is an error; no local
node is started in its place.</p>
<p><strong>Using a node you started yourself.</strong> The engine restores a snapshot only from
its own data directory or from a directory named in
<code class="inline">[limits] snapshot_repo_allowlist</code>. Packs are extracted under
<code class="inline">~/.xerj/hub/packs</code>, so a node started without that setting refuses the
restore:</p>
<pre class="code">$ xerj hub pull lz4
xerj hub pull: restore snapshot: POST /_snapshot/hub-lz4/lz4-0774d055/_restore?wait_for_completion=true -> HTTP 400: {"error":{…"reason":"snapshot repository location [/home/you/.xerj/hub/packs/lz4/0774d055/snapshot] is outside data_dir and not in limits.snapshot_repo_allowlist; refusing (set the allowlist to permit external snapshot roots)"…}</pre>
<p>Fix: add the pack directory to the node's config file, as an absolute path (TOML does not
expand <code class="inline">~</code>), and restart the node with <code class="inline">--config</code>:</p>
<pre class="code">$ printf '[limits]\nsnapshot_repo_allowlist = ["%s/.xerj/hub/packs"]\n' "$HOME" > ~/xerj-hub.toml
$ nohup xerj --insecure --data-dir ~/xerj-data --config ~/xerj-hub.toml > ~/xerj-node.log 2>&1 & echo $! > ~/xerj-node.pid
$ xerj hub pull lz4
pulled lz4 0774d055 5810327 → 2 indices, 1702 definitions</pre>
<p>Verify: the last line starts with <code class="inline">pulled lz4</code> (or
<code class="inline">lz4: already present</code> on a second run). Stop that node with
<code class="inline">kill $(cat ~/xerj-node.pid)</code>. For a node with auth on, export
<code class="inline">XERJ_API_KEY</code> (the key is in <code class="inline"><data-dir>/admin.key</code>).</p>
<p><strong><code class="inline">xerj code</code> does not start a node.</strong>
<code class="inline">xerj code <slug> --def <SYMBOL></code> and
<code class="inline">xerj code <slug> "<query>"</code> read a pulled pack too, but
only from <code class="inline">--url</code>, <code class="inline">XERJ_URL</code> or
<code class="inline">localhost:9200</code>. If the pack lives in the node
<code class="inline">xerj ask</code> started, pass its port (from
<code class="inline">xerj hub status</code>), or use <code class="inline">xerj ask --corpus <slug></code>:</p>
<pre class="code">$ xerj code lz4 --def LZ4_MAX_INPUT_SIZE --url http://127.0.0.1:9300
lz4 pin 2026-10-02 (15 days old) mixed: BSD-2 (lib) / GPL-2.0+ (rest) lz4/lib/lz4.h:214
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
value: 0x7E000000
approach-only licence (mixed: BSD-2 (lib) / GPL-2.0+ (rest)): read, do not copy
answered from: local pack</pre>
<p>Verify: exit 0 and a <code class="inline">value:</code> line. Pointed at a node without the
pack, the same command prints <code class="inline">defs query failed: …</code> and
<code class="inline">no match in lz4</code>, and exits 1.</p>
<h2 id="network">Offline, no download, and the size cap</h2>
<pre class="code">$ xerj ask --offline --corpus snappy "varint decoding"
no local pack for 'snappy' and --offline forbids downloading it; run: xerj hub pull snappy (14.3 MB)
$ xerj ask --no-fetch --corpus zstd --def ZSTD_CLEVEL_DEFAULT
no local pack for 'zstd' and --no-fetch forbids downloading it; run: xerj hub pull zstd (22.1 MB)</pre>
<p>Both exit 1. <code class="inline">--offline</code> never touches the network: it reads the
cached catalogue and cached packs only, and a corpus already on disk answers normally.
<code class="inline">--no-fetch</code> does not download a pack. Run the printed line, then ask
again:</p>
<pre class="code">$ xerj hub pull snappy
pulled snappy 9c28114a 15034490 → 3 indices, 382 definitions</pre>
<p>Verify: <code class="inline">pulled <slug> <pin8> <bytes> → <n> indices, <defs> definitions</code>
and exit 0.</p>
<p>For a pack over the 50 MiB auto-pull cap, <code class="inline">xerj ask</code> prints
<code class="inline">'<slug>' is <MB> MB, over the 50.0 MB auto-pull cap, so a question does not download it automatically and no hosted answer is available; run `xerj hub pull <slug>` to fetch it once, then ask again</code>
and exits 1. The explicit <code class="inline">xerj hub pull <slug></code> does fetch it, bounded by the 4 GiB disk budget (<code class="inline">DISK_BUDGET_BYTES</code>), so the corpus is usable once pulled; only a pack larger than the whole disk budget is refused.</p>
<h2 id="hosted">The hosted setting</h2>
<p><code class="inline">--hosted on|off|auto</code> on <code class="inline">xerj ask</code>, or
<code class="inline">xerj hub config hosted on|off|auto</code> stored in
<code class="inline">~/.xerj/hub/config.toml</code>, decides whether a question may be sent to a
hosted hub node. <code class="inline">auto</code>, the default, is off inside a git repository
(the first time, XERJ prints <code class="inline">hosted answers are off in git repositories; xerj hub config hosted on</code>)
and on elsewhere. No public hosted node exists, and this build has no hosted client, so every
answer comes from a local pack whatever the setting.</p>
<pre class="code">$ xerj hub config hosted off
hosted = off</pre>
<p>Verify: <code class="inline">xerj hub status</code> shows <code class="inline">hosted: off</code>.</p>
<h2 id="mcp">From an MCP client</h2>
<p><code class="inline">xerj mcp</code> serves three hub tools next to its other tools. They run
the same flow as <code class="inline">xerj ask</code>, through the node that answers on
<code class="inline">--url</code> / <code class="inline">XERJ_URL</code> or else the node XERJ
starts:</p>
<pre class="code">xerj_hub_ask {question, corpus?, mode?: "def"|"how"|"files", max_tokens?} one turn, like xerj ask
xerj_hub_def {slug, symbol, k?} like xerj ask --corpus <slug> --def <SYMBOL>
xerj_hub_find {question, k?} which corpora could answer; no passages</pre>
<p><strong>A hub tool never downloads a pack and never fetches the catalogue.</strong> A tool call
must not wait on the network. Run <code class="inline">xerj hub sync</code> in a shell once, and
<code class="inline">xerj hub pull <slug></code> for each corpus, then call the tool. Until
then the tool returns an error (<code class="inline">isError: true</code>) that says which command
to run:</p>
<pre class="code">no hub catalogue is cached on this machine, and hub tools never fetch one (a tool call does not wait on the network); run `xerj hub sync` in a shell once, then call the tool again (…)
no local pack for 'lz4' and --no-fetch forbids downloading it; run: xerj hub pull lz4 (5.5 MB)</pre>
<p>With the pack on disk, <code class="inline">xerj_hub_def {"slug":"lz4","symbol":"LZ4_VERSION_RELEASE"}</code>
returns the same text as the <a href="#reading">shell answer</a>, and the structured content is the
answer JSON. <code class="inline">xerj_hub_find {"question":"XXH64"}</code> returns:</p>
<pre class="code">lz4 — an agent needing fast block compression queries this for the block format and decoder loop (score 0.45, pin 2026-10-02, mixed: BSD-2 (lib) / GPL-2.0+ (rest), cached)
zstd — an agent tuning compression queries this for frame/block formats, window and level internals (score 0.45, pin 2026-10-02, mixed: BSD-3 (lib) / GPL-2.0 (cli), cached)
routing: ambiguous; ask one with xerj_hub_ask {question, corpus}</pre>
<p><strong>Register it once</strong>, with the absolute path of the binary (an editor started from
a desktop icon does not inherit your shell's <code class="inline">PATH</code>;
<code class="inline">command -v xerj</code> prints it), then restart the session:</p>
<pre class="code"># Claude Code
claude mcp add --scope user xerj -- "$(command -v xerj)" mcp
# .mcp.json (Claude Code project scope; `xerj init` writes it), Cursor, Windsurf,
# Claude Desktop, Cline: root key mcpServers
{"mcpServers": {"xerj": {"command": "/abs/path/to/xerj", "args": ["mcp"]}}}
# VS Code, .vscode/mcp.json: root key servers (a block under mcpServers is ignored)
{"servers": {"xerj": {"type": "stdio", "command": "/abs/path/to/xerj", "args": ["mcp"]}}}</pre>
<p>Verify without a client: this prints <code class="inline">xerj_hub_ask</code> once.</p>
<pre class="code">$ printf '%s\n' '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"check","version":"0"}}}' \
'{"jsonrpc":"2.0","id":2,"method":"tools/list"}' | xerj mcp 2>/dev/null | grep -o '"name":"xerj_hub_ask"'
"name":"xerj_hub_ask"</pre>
<p>A build with no hub key serves 19 tools and omits the three hub tools. The full schemas are in
<a href="/docs/agents/schemas/mcp-tools.json">mcp-tools.json</a>; the
<a href="/docs/agents/endpoints#hub-tools">endpoint contract</a> lists them with the other agent
operations.</p>
<h2 id="sync">Keeping packs current</h2>
<pre class="code">$ xerj hub sync
hub sync: catalogue refreshed (generated 2026-10-17T23:35:36Z, 3 corpora)
hub sync: lz4 0774d055: current
hub sync: 0 updated, 1 current, 0 kept, 0 failed, 0 pruned</pre>
<p><code class="inline">xerj hub sync</code> refreshes the catalogue, pulls the new pin of every
cached pack whose pin moved (sha256 and signature are checked first), removes the old pin only
after the new one is restored, prunes to the disk budget, and writes
<code class="inline">~/.xerj/hub/last_sync</code>. A failed update keeps the old pin serving.
Verify: exit 0 and <code class="inline">0 failed</code> on the last line.</p>
<pre class="code">$ xerj hub sync --install-timer
/home/you/.config/systemd/user/xerj-hub-sync.timer</pre>
<p>This writes a systemd user timer and service (Linux) or a launchd agent (macOS) that runs
<code class="inline">xerj hub sync</code> every 6 hours, and prints the path. It enables, loads
and starts nothing: the comment at the top of the written file says how. Verify: the printed file
exists. Hub packs are never refused as stale; the age is in every answer's header, and a newer pin
arrives with <code class="inline">sync</code>.</p>
<h2 id="exit-codes">Exit codes</h2>
<pre class="code">xerj ask 0 answered · 1 no match, nothing to answer from, or a pull/node failure
2 usage error or unknown corpus · 3 a pack failed verification
4 more than one corpus matches (candidates printed; re-run with --corpus)
5 this build has no hub public key: nothing was downloaded
xerj hub pull 0 pulled or already present · 1 anything else (excluded licence and
over-the-cap packs included) · 2 unknown corpus · 3 verification failed
5 no hub public key
xerj hub sync 0 synced or nothing to do · 1 catalogue not refreshed or a pack not
updated (old pin kept) · 2 usage error · 3 a pack failed verification</pre>
<p>Exit 3 means the bytes did not match: nothing was extracted, and a retry downloads the same
bytes. Report it rather than retrying. Exit 5 cannot be fixed by a retry either.</p>
<h2 id="files">Where things live</h2>
<pre class="code">~/.xerj/hub/catalogue.json, .sig the cached, signed catalogue (refreshed after 24 h, or by sync)
~/.xerj/hub/packs/<slug>/<pin8>/ extracted packs (pack.json, SHA256SUMS, snapshot/, last_used)
~/.xerj/hub/node/ data of the node xerj ask starts; node.log, node.pid,
node.port and node.toml beside it
~/.xerj/hub/config.toml the hosted setting
~/.xerj/hub/last_sync written by a sync with no failures</pre>
<p><code class="inline">XERJ_HUB_HOME</code> moves the whole directory.
<code class="inline">XERJ_HUB_URL</code> points the client at another hub (default
<code class="inline">https://hub.xerj.org</code>; the catalogue is read from
<code class="inline"><hub>/api/v1/catalogue.json</code> and its <code class="inline">.sig</code>).
Both are listed on the <a href="/docs/env#client-env">environment page</a>.</p>
<h2 id="troubleshooting">When it does not work</h2>
<p><strong>The catalogue is unavailable.</strong> A static host can answer an unknown path with an
HTML page and status 200. The client checks the content and the signature, and refuses it:</p>
<pre class="code">$ xerj hub sync
xerj hub sync: cannot refresh the catalogue; nothing was changed: the hub catalogue is unavailable: https://hub.xerj.org/api/v1/catalogue.json returned 264903 bytes starting "<!doctype html><html lang=\"en\"><head><me" (text/html; charset=utf-8), not a signed catalogue; no catalogue is served at this address</pre>
<p>Exit 1, and a cached catalogue (if any) is kept. <code class="inline">xerj ask</code> and
<code class="inline">xerj hub pull</code> print the same reason after
<code class="inline">cannot load the hub catalogue:</code>. Packs already on disk still answer with
<code class="inline">--offline</code>.</p>
<p><strong>A pack or the catalogue fails verification.</strong></p>
<pre class="code">$ xerj hub pull snappy
xerj hub pull: pack verification failed: pack sha256 mismatch: archive is 97351ed6…, expected ae10dc7d…
$ xerj hub sync
xerj hub sync: cannot refresh the catalogue; nothing was changed: catalogue signature did not verify; refusing to replace the cache: …</pre>
<p>The pull exits 3 and extracts nothing; the sync exits 1 and keeps the previous catalogue.
(Both were produced by changing one byte of a served file.)</p>
<p><strong>No hub key in this build.</strong></p>
<pre class="code">$ xerj ask --corpus snappy "varint"
pull of 'snappy' failed: no hub public key is configured in this build (or XERJ_HUB_PUBKEY=none), so a pack's signature cannot be verified; nothing was downloaded</pre>
<p>Exit 5. Use a binary that has the key (see the check at the top of this page). For your own
files, <code class="inline">xerj autoindex</code> works in every build.</p>
<h2 id="contribute">Contributing a corpus</h2>
<p>The registry lives on the
<a href="https://github.com/xerj-org/xerj/tree/corpus-hub" rel="external"><code class="inline">corpus-hub</code>
branch</a> of xerj-org/xerj, and contributions arrive as pull requests against that branch. Its
<a href="https://github.com/xerj-org/xerj/blob/corpus-hub/tools/xerj-code/hub/CONTRIBUTING.md" rel="external">CONTRIBUTING.md</a>
is the source of record for the rules; this section is the walkthrough. Browse the registry at
<a href="https://hub.xerj.org" rel="external">hub.xerj.org</a>.</p>
<p>You contribute a <strong>build input</strong>, never a pack. A manifest or a recipe goes into
git. CI (<code class="inline">.github/workflows/hub-packs.yml</code>) builds the pack with
<code class="inline">xerj hub build-pack</code>, signs it with the hub key, pulls it back on a
fresh node and checks one <code class="inline">--def</code> and one question before it publishes
the release, then rebuilds and signs the catalogue with
<code class="inline">xerj hub build-catalogue</code>. Users download the signed pack; they do not
rebuild the corpus from your manifest.</p>
<h3 id="lanes">Two lanes</h3>
<pre class="code">LANE A — reference corpus tools/xerj-code/hub/<name>.json (a manifest)
LANE B — record pack tools/packs/<name>/recipe.toml (a recipe)</pre>
<p>A <strong>reference corpus</strong> is other people's source code at pinned commits, the
material for reference coding ("how do three other engines implement merge policy?"). A
<strong>record pack</strong> is structured data you curate and resolve, for example Rust
vulnerability advisories identity-merged across osv.dev and RustSec. If your content is "files
someone can clone", it is Lane A; if it is "records distilled from one or more sources", it is
Lane B. The catalogue that <code class="inline">xerj ask</code> reads lists Lane A corpora; the
record packs are published separately by <code class="inline">pack-publish.yml</code>
(<a href="/docs/corpus-hub#lane-b">below</a>).</p>
<h3 id="lane-a">Lane A — the manifest</h3>
<p>Copy <code class="inline">tools/xerj-code/hub/TEMPLATE.json</code> to
<code class="inline">tools/xerj-code/hub/<your-corpus>.json</code>. The corpus name must equal
the filename. Field by field:</p>
<pre class="code">{
"corpus": "xerj-storage",
"cloned_at": "<UTC timestamp the build ran>",
"repos": [
{
"repo": "sled",
"url": "https://github.com/spacejam/sled",
"licence": "<what the detector read — do not trust this alone>",
"sha": "<full 40-hex commit the build actually used>",
"files": 417, "bytes": 2900000,
"review": {
"spdx": "<SPDX id YOU confirmed by opening the licence file>",
"use": "adapt-with-attribution",
"by": "your-handle", "at": "2026-10-01",
"note": "<one line: what you opened, what you concluded>"
}
}
]
}</pre>
<p>The two fields CI cannot check, and a reviewer will hold you to: <code class="inline">sha</code>
must be the commit the build <em>actually used</em> (never a branch head), and
<code class="inline">review</code> must be a human's verdict (see
<a href="#licence">the licence contract</a>). The <code class="inline">sha</code> becomes the
pack's pin, the <code class="inline">pin</code> every answer prints.</p>
<h3 id="lane-b">Lane B — the recipe</h3>
<p>Copy <code class="inline">tools/packs/TEMPLATE-recipe.toml</code> to
<code class="inline">tools/packs/<name>/recipe.toml</code>. Everything domain-specific lives
in the recipe, never in the tooling:</p>
<pre class="code">[recipe] format = 1 · name = directory name · description
[[sources]] one per source: slug, kind (git|http-zip|dir), url,
glob, format (flat|osv|rustsec-md), licence
[envelope] how a raw source file becomes one searchable record:
id_from, title_from, body_join, defs_from
[identity] which fields make two records the same thing
(edges, each=true explodes list fields; union-find
closes the chain) + canonical_source_order
[merge] precedence when two sources carry the same fact
[[derived]] optional build-time fields (regex_extract, present)
[emit] shards</pre>
<p>The identity block is the whole dedup contract: write it so a reader can predict the merged
count, then put the measured count (envelopes → records) in your README. The showcase to read
alongside the template is <code class="inline">tools/packs/rust-vulns/recipe.toml</code>. Built
packs are <strong>never committed</strong>; a maintainer builds from your recipe and attaches the
signed pack to a GitHub Release.</p>
<h3 id="licence">The licence contract</h3>
<p>Every source carries a human review block with one of three verdicts:</p>
<pre class="code">adapt-with-attribution Apache-2.0 · MIT · BSD — cite file:line when
you adapt; copying is fine within the licence
approach-only AGPL · SSPL · Elastic · BUSL · GPL — read the
design, write your own code, never paste
mixed per-file split (e.g. MIT core + BUSL edition) —
name the boundary in the note</pre>
<p>The verdict is what <code class="inline">xerj ask</code> prints: an approach-only or mixed
corpus carries <code class="inline">read, do not copy</code> on every hit. Our licence detector once
reported Elasticsearch as Apache-2.0, because the AGPL text contains the phrase «an "Apache License
2.0" compatible license». A detector's false "permissive" is how restricted code ends up pasted
into an Apache-2.0 project. CI checks that a review block <em>exists</em> and that its fields are
well-formed; only the reviewer's own reading of the licence file makes it true.</p>
<h3 id="pr">The pull request</h3>
<pre class="code"># 1. branch FROM corpus-hub (not main) so the CI workflow is in your PR
git clone -b corpus-hub https://github.com/xerj-org/xerj && cd xerj
# 2. add your manifest or recipe, then validate locally (stdlib only)
python3 tools/xerj-code/hub/validate_hub.py
# 3. commit, push, open the PR against corpus-hub
git checkout -b corpus/<your-name>
git add tools/xerj-code/hub/<your-name>.json
git commit -m "corpus: add <your-name> (<domain>)"
git push -u origin corpus/<your-name></pre>
<p>CI on every push and PR runs the same validator plus two registry guards: no file over 5 MB
under the registry paths (built packs and key material must not be in git), and no private key
material under <code class="inline">tools/packs/keys</code>. A maintainer then reviews domain fit,
pin truthfulness and, above all, the licence verdicts, and merges. After the merge, CI builds the
pack; a pack whose consumer check fails stays a draft and is not catalogued.</p>
<h3 id="fit">Is your corpus a fit?</h3>
<p><strong>1. Can you name the domain?</strong> Finish the sentence: "an agent working on ___
would query this for ___." That sentence becomes the corpus's <code class="inline">ask_for</code>
line, which <code class="inline">xerj ask</code> prints when it lists candidates. A corpus that
contains everything retrieves like a search engine with no query.</p>
<p><strong>2. Is the content un-memorised?</strong> Retrieval helps most on niche, private,
internal and post-cutoff material. On famous public library code a model already knows, retrieval
is overhead. Do not force it where it loses.</p>
<p><strong>Your own corpora.</strong> For a set of repositories only you need,
<code class="inline">xerj corpus add <name> <git-url>…</code> (or
<code class="inline">--from</code> a manifest file you wrote) and
<code class="inline">xerj corpus index <name></code> still clone and index on your machine,
and <code class="inline">xerj code <name></code> searches the result. That is a local path
and does not use the hub.</p>
<div class="next">
<div class="step"><span class="dir">◀ PREV</span><a href="/docs/cli">CLI</a></div>
<div class="step right"><span class="dir">NEXT ▶</span><a href="/docs/recipes/">Recipes</a></div>
</div>
<!-- Print-only branded footer (hidden on screen, visible in PDF) -->
<div class="print-footer" style="display:none;">
<span>XERJ.AI · V1.0.0-RC.100 · 2026-10-17</span>
<a href="https://xerj.org/docs/">https://xerj.org/docs/</a>
<span>Corpus Hub</span>
</div>
</main>
</div>
<!-- seo:footer-begin — generated by scripts/seo/fix_links.py · do not hand-edit -->
<nav class="footer-map" aria-label="Site map">
<div class="fm-col">
<span class="fm-h">PRODUCT</span>
<a href="/product">PRODUCT</a>
<a href="/architecture/">ARCHITECTURE</a>
<a href="/benchmarks/">BENCHMARKS</a>
<a href="/pricing/">PRICING</a>
<a href="/security/">SECURITY</a>
<a href="/playground">DASHBOARDS</a>
<a href="/demo/">REAL-DATA DEMO</a>
<a href="/aise-demo">PRODUCT TOUR</a>
</div>
<div class="fm-col">
<span class="fm-h">LEARN</span>
<a href="/answers/">ANSWERS</a>
<a href="/compare/">COMPARE</a>
<a href="/use-cases">USE CASES</a>
<a href="/case-studies">CASE STUDIES</a>
<a href="/industries/">INDUSTRIES</a>
<a href="/blog/">BLOG</a>
</div>
<div class="fm-col">
<span class="fm-h">SOLUTIONS</span>
<a href="/solutions/">SOLUTIONS</a>
<a href="/industries/finserv">FINANCIAL SERVICES</a>
<a href="/industries/public-sector">PUBLIC SECTOR</a>
<a href="/industries/healthcare">HEALTHCARE</a>
<a href="/industries/retail">RETAIL</a>
<a href="/use-cases/code-security-audit">AI CODE-SECURITY AUDIT</a>
<a href="/use-cases/semantic-analytics">SEMANTIC ANALYTICS</a>
<a href="/resources/">SALES KIT</a>
</div>
<div class="fm-col">
<span class="fm-h">DOCS</span>
<a href="/docs/">DOCUMENTATION</a>
<a href="/docs/quickstart">QUICKSTART</a>
<a href="/docs/install">INSTALL</a>
<a href="/docs/migration-from-es">MIGRATE FROM ELASTICSEARCH</a>
<a href="/docs/recipes/">RECIPES</a>
<a href="/docs/agents/">DOCS FOR AI</a>
<a href="/for-agents">FOR AI AGENTS</a>
<a href="/agent-search/">AGENTIC SEARCH DEMO</a>
<a href="/llms.txt">LLMS.TXT</a>
<a href="https://github.com/xerj-org/xerj" rel="external">GITHUB</a>
</div>
</nav>
<!-- seo:footer-end -->
<footer class="footer">
<span>XERJ.AI · DOCS · V1.0.0-RC.100</span>
<span>READ THE SOURCE</span>
<span><a href="/">HOME</a> · <a href="/product">PRODUCT</a> · <a href="/answers/">ANSWERS</a> · <a href="/playground">DASHBOARDS</a> · <a href="/llms.txt">LLMS.TXT</a></span>
</footer>
<script type="application/json" id="docs-index">[{"title":"Corpus Hub","slug":"/docs/corpus-hub","kicker":"CORPUS HUB","snip":"Ask a pinned, signed, pre-indexed corpus: xerj ask, xerj hub pull/sync, the MCP hub tools, and how to contribute a corpus."},{"title":"Overview","slug":"/docs/","kicker":"START","snip":""},{"title":"Quickstart","slug":"/docs/quickstart","kicker":"START","snip":""},{"title":"Install","slug":"/docs/install","kicker":"START","snip":""},{"title":"Migrate from ES","slug":"/docs/migration-from-es","kicker":"START","snip":""},{"title":"CLI","slug":"/docs/cli","kicker":"REFERENCE","snip":""},{"title":"Config TOML","slug":"/docs/config","kicker":"REFERENCE","snip":""},{"title":"Environment","slug":"/docs/env","kicker":"REFERENCE","snip":""},{"title":"Native REST API","slug":"/docs/api-native","kicker":"REFERENCE","snip":""},{"title":"ES-compat API","slug":"/docs/api-es-compat","kicker":"REFERENCE","snip":""},{"title":"Query types","slug":"/docs/queries","kicker":"DATA MODEL","snip":""},{"title":"Aggregations","slug":"/docs/aggregations","kicker":"DATA MODEL","snip":""},{"title":"Analyzers","slug":"/docs/analyzers","kicker":"DATA MODEL","snip":""},{"title":"Vectors & kNN","slug":"/docs/vectors","kicker":"DATA MODEL","snip":""},{"title":"Ingest pipelines","slug":"/docs/ingest","kicker":"ENGINE","snip":""},{"title":"Storage & WAL","slug":"/docs/storage","kicker":"ENGINE","snip":""},{"title":"Compression","slug":"/docs/compression","kicker":"ENGINE","snip":""},{"title":"Clustering","slug":"/docs/clustering","kicker":"ENGINE","snip":""},{"title":"Metrics","slug":"/docs/metrics","kicker":"ENGINE","snip":""},{"title":"Auth & TLS","slug":"/docs/security","kicker":"ENGINE","snip":""},{"title":"Running in production","slug":"/docs/operations","kicker":"OPERATIONS","snip":""},{"title":"Backup & restore","slug":"/docs/backup-restore","kicker":"OPERATIONS","snip":""},{"title":"Upgrades","slug":"/docs/upgrades","kicker":"OPERATIONS","snip":""},{"title":"Troubleshooting","slug":"/docs/troubleshooting","kicker":"OPERATIONS","snip":""},{"title":"SIEM","slug":"/docs/playbooks/siem","kicker":"PLAYBOOKS","snip":""},{"title":"Log analytics","slug":"/docs/playbooks/log-analytics","kicker":"PLAYBOOKS","snip":""},{"title":"Vector · RAG","slug":"/docs/playbooks/vector-search","kicker":"PLAYBOOKS","snip":""},{"title":"Full-text search","slug":"/docs/playbooks/full-text","kicker":"PLAYBOOKS","snip":""},{"title":"Observability","slug":"/docs/playbooks/observability","kicker":"PLAYBOOKS","snip":""},{"title":"CLI · --config","slug":"/docs/cli#config-flag","kicker":"CLI","snip":"-c, --config <PATH> Path to the TOML config file."},{"title":"CLI · --data-dir","slug":"/docs/cli#data-dir-flag","kicker":"CLI","snip":"-d, --data-dir <PATH> Override [server] data_dir at the command line."},{"title":"CLI · --insecure","slug":"/docs/cli#insecure-flag","kicker":"CLI","snip":"-k, --insecure Disable TLS and API-key auth. Dev only."},{"title":"[server] rest_port","slug":"/docs/config#server","kicker":"CONFIG","snip":"u16 · default 8080 · Native REST API listener port."},{"title":"[server] es_compat_port","slug":"/docs/config#server","kicker":"CONFIG","snip":"u16 · default 9200 · Elasticsearch-wire-format port. 0 = OS-assigned port."},{"title":"[storage] wal_sync","slug":"/docs/config#storage","kicker":"CONFIG","snip":"enum · default batched · sync · batched · async. Durability knob."},{"title":"[vector] hnsw_ef_search","slug":"/docs/config#vector","kicker":"CONFIG","snip":"u32 · default 100 · Accepted but not wired — the beam width comes from the request num_candidates."},{"title":"[vector] default_quantization","slug":"/docs/config#vector","kicker":"CONFIG","snip":"enum · none (default) · scalar8. binary is refused at startup; scalar4 is not accepted."},{"title":"GET /v1/health","slug":"/docs/api-native#health","kicker":"NATIVE API","snip":"Liveness + readiness. {status, uptime_s, version}."},{"title":"GET /v1/metrics","slug":"/docs/api-native#metrics","kicker":"NATIVE API","snip":"Prometheus text format. Scrape-friendly."},{"title":"POST /v1/indices/:name/turbo-ingest","slug":"/docs/api-native#turbo-ingest","kicker":"NATIVE API","snip":"NDJSON bulk path with parallel tokenization."},{"title":"POST /v1/indices/:name/search","slug":"/docs/api-native#search","kicker":"NATIVE API","snip":"Unified query endpoint — full-text, term, range, KNN, hybrid."},{"title":"POST /:index/_bulk","slug":"/docs/api-es-compat#bulk","kicker":"ES API","snip":"NDJSON bulk. Actions: index, create, update, delete."},{"title":"POST /:index/_search","slug":"/docs/api-es-compat#search","kicker":"ES API","snip":"ES query DSL. Drop-in migration path."},{"title":"match_all","slug":"/docs/queries#match-all","kicker":"QUERY","snip":"Everything in the index."},{"title":"bool","slug":"/docs/queries#bool","kicker":"QUERY","snip":"must / should / must_not / filter clauses."},{"title":"knn","slug":"/docs/queries#knn","kicker":"QUERY","snip":"Vector kNN — HNSW-served when unfiltered (exact-rescored); filtered runs the exact scan."},{"title":"semantic","slug":"/docs/queries#semantic","kicker":"QUERY","snip":"Embed the query text at query time and search the vector field."},{"title":"hybrid","slug":"/docs/queries#hybrid","kicker":"QUERY","snip":"RRF / linear fusion of sub-queries."},{"title":"terms aggregation","slug":"/docs/aggregations#terms","kicker":"AGG","snip":"Exact, pre-computed terms histograms over keyword fields."},{"title":"date_histogram","slug":"/docs/aggregations#date-histogram","kicker":"AGG","snip":"Time-bucketed counts."},{"title":"kNN serving","slug":"/docs/vectors#knn-query","kicker":"VECTORS","snip":"Unfiltered kNN via persisted HNSW + exact rescore; filtered kNN exact scan."},{"title":"SQ8 quantization","slug":"/docs/vectors#quantization","kicker":"VECTORS","snip":"8-bit scalar quantization. int8 scoring precision, not a memory saving (#392)."},{"title":"turbo-ingest","slug":"/docs/ingest#turbo","kicker":"INGEST","snip":"NDJSON bulk path. Parallel tokenization across cores."},{"title":"delta-of-delta","slug":"/docs/compression#delta-of-delta","kicker":"COMPRESSION","snip":"Timestamp encoding. Apache/Nginx/ISO auto-detect."},{"title":"dictionary encoding","slug":"/docs/compression#dictionary","kicker":"COMPRESSION","snip":"Up to 256 unique strings per column."},{"title":"api_key","slug":"/docs/security#api-key","kicker":"SECURITY","snip":"Static admin key. Auto-generated on first run."},{"title":"SIEM playbook","slug":"/docs/playbooks/siem","kicker":"PLAYBOOK","snip":"Terms agg over source IPs for SIEM, with detection rules."},{"title":"RAG playbook","slug":"/docs/playbooks/vector-search","kicker":"PLAYBOOK","snip":"Embeddings, kNN, hybrid search, BM25 + kNN fusion."},{"title":"[cluster] enabled","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"bool · default false · Enable multi-node Raft cluster mode."},{"title":"[cluster] peers","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"array · \"host:port=host:port\" · label must equal address (#1171)."},{"title":"[cluster] port","slug":"/docs/config#cluster","kicker":"CONFIG","snip":"u16 · default 9300 · Intra-cluster Raft + search port."},{"title":"[embedding] default_endpoint","slug":"/docs/config#embedding","kicker":"CONFIG","snip":"string · OpenAI-compatible embeddings URL for auto-embedding on ingest."},{"title":"[embedding] default_model","slug":"/docs/config#embedding","kicker":"CONFIG","snip":"string · Model name (e.g. text-embedding-3-small, nomic-embed-text)."},{"title":"Raft","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Embedded Raft ring — leader election, heartbeats, authenticated transport."},{"title":"Leader failover","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Kill the leader of a 3-node ring: survivors elect a new leader in ~250 ms."},{"title":"Cluster honesty","slug":"/docs/clustering","kicker":"CLUSTER","snip":"Create-time number_of_replicas copies acknowledged document writes. Bulk, replica promotion, and snapshot catch-up are still open (#1170)."},{"title":"Systemd unit","slug":"/docs/operations","kicker":"OPS","snip":"Production unit file with hardening directives."},{"title":"Readiness probe","slug":"/docs/operations","kicker":"OPS","snip":"GET /v1/health/ready — only 200 once WAL replay is complete."},{"title":"Config reload","slug":"/docs/operations","kicker":"OPS","snip":"SIGHUP reloads most keys without restart."},{"title":"Hot backup","slug":"/docs/backup-restore","kicker":"OPS","snip":"rsync the data dir after POST /v1/admin/flush."},{"title":"S3 backup","slug":"/docs/backup-restore","kicker":"OPS","snip":"POST /v1/admin/backup → any S3-compatible endpoint."},{"title":"Cluster upgrade","slug":"/docs/upgrades","kicker":"OPS","snip":"The cluster wire has no version negotiation — full stop/start, not rolling."},{"title":"Merge pressure","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Segment count climbing → raise merge.max_concurrent + io_rate."},{"title":"Cluster flapping","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Sustained leader churn is a defect — capture RUST_LOG=xerj_cluster=debug and file it."},{"title":"WAL warnings after a crash","slug":"/docs/troubleshooting","kicker":"OPS","snip":"Repaired on startup; no manual repair command."},{"title": "Recipes", "slug": "/docs/recipes/", "kicker": "RECIPES", "snip": "Task-oriented AI & search recipes, each verified end-to-end against a live XERJ."},{"title": "Semantic search & RAG", "slug": "/docs/recipes/semantic-search-rag", "kicker": "RECIPE", "snip": "Retrieval by meaning with semantic_text — auto-embed on ingest, no separate vector DB."},{"title": "Agent memory", "slug": "/docs/recipes/agentic-memory", "kicker": "RECIPE", "snip": "Long-term memory for agents via /_memory — store, recall, forget, per-agent isolation."},{"title": "Passage retrieval", "slug": "/docs/recipes/passage-retrieval", "kicker": "RECIPE", "snip": "Auto-chunk long docs; best-passage (max-sim) scoring — 98% top-3."},{"title": "Vector search · kNN", "slug": "/docs/recipes/vector-search-knn", "kicker": "RECIPE", "snip": "Nearest-neighbor over dense_vector — HNSW-served unfiltered, exact with filters."},{"title": "Vector quantization", "slug": "/docs/recipes/vector-quantization", "kicker": "RECIPE", "snip": "scalar8 (accepts ES int8_hnsw; exact code scan) — int8 scoring at recall@10 = 0.998. Precision, not memory."},{"title": "Hybrid search", "slug": "/docs/recipes/hybrid-search", "kicker": "RECIPE", "snip": "Keyword + vector in one query — RRF fusion."},{"title": "Log analytics", "slug": "/docs/recipes/log-analytics", "kicker": "RECIPE", "snip": "Raw logs to dashboards — error rates, p95 latency, top services."},{"title": "Anomaly detection", "slug": "/docs/recipes/anomaly-detection", "kicker": "RECIPE", "snip": "Statistical _ml detectors that flag spikes in metrics/logs."},{"title": "Anomaly datafeeds", "slug": "/docs/recipes/continuous-anomaly-datafeeds", "kicker": "RECIPE", "snip": "Live _ml datafeed re-scores an index on a timer, stores anomaly records."},{"title": "Migrate from ES", "slug": "/docs/recipes/migrate-from-elasticsearch", "kicker": "RECIPE", "snip": "Point your existing ES client at XERJ — same wire, change the URL."}]</script>
<script>
// IIFE — js-auth navigates between decrypted pages in the same
// JS realm, so const bindings must be lexically scoped.
(function () {
const root = document.documentElement;
// Client-side docs search — filters the inline JSON index.
const searchEl = document.getElementById('docsSearch');
const inputEl = document.getElementById('docsSearchInput');
const resultsEl = document.getElementById('docsSearchResults');
const indexJson = document.getElementById('docs-index');
if (searchEl && inputEl && resultsEl && indexJson) {
let index = [];
try { index = JSON.parse(indexJson.textContent || '[]'); } catch {}
const render = (q) => {
const query = q.trim().toLowerCase();
if (!query) { searchEl.classList.remove('active'); resultsEl.innerHTML = ''; return; }
const scored = [];
for (const e of index) {
const hay = (e.title + ' ' + (e.kicker || '') + ' ' + (e.snip || '')).toLowerCase();
if (hay.includes(query)) {
let score = 0;
if (e.title.toLowerCase().includes(query)) score += 10;
if ((e.kicker || '').toLowerCase() === query.toUpperCase()) score += 5;
if ((e.snip || '').toLowerCase().includes(query)) score += 1;
scored.push({ e, score });
}
}
scored.sort((a, b) => b.score - a.score);
const top = scored.slice(0, 20);
if (top.length === 0) {
resultsEl.innerHTML = '<div class="no-results">NO MATCHES — TRY A SHORTER QUERY</div>';
} else {
resultsEl.innerHTML = top.map(({e}) => {
const kicker = e.kicker || '';
const snip = e.snip ? '<div class="snip">' + escHtml(e.snip) + '</div>' : '';
return '<a href="' + e.slug + '">' +
'<span class="kicker">' + escHtml(kicker) + '</span>' +
'<span class="title">' + escHtml(e.title) + '</span>' +
snip + '</a>';
}).join('');
}
searchEl.classList.add('active');
};
const escHtml = (s) => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
inputEl.addEventListener('input', (e) => render(e.target.value));
inputEl.addEventListener('focus', () => { if (inputEl.value) render(inputEl.value); });
document.addEventListener('keydown', (e) => {
if ((e.key === 'k' || e.key === 'K') && (e.metaKey || e.ctrlKey)) {
e.preventDefault();
inputEl.focus();
inputEl.select();
}
if (e.key === 'Escape' && document.activeElement === inputEl) {
inputEl.value = '';
searchEl.classList.remove('active');
resultsEl.innerHTML = '';
inputEl.blur();
}
});
}
})();
</script>
</body>
</html>