Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Extract root index HTML from Makefile into standalone file (#767)

Louis Choquel6cef4e5f8b87
diff --git a/.github/workflows/doc-check.yml b/.github/workflows/doc-check.yml
index 45a58e877..088d26ee3 100644
--- a/.github/workflows/doc-check.yml
+++ b/.github/workflows/doc-check.yml
@@ -34,7 +34,7 @@ jobs:
- name: Install mkdocs
run: |
python -m pip install --upgrade pip
- pip install mkdocs==1.6.1 mkdocs-material==9.6.14 mkdocs-glightbox==0.4.0 mkdocs-meta-manager==1.1.0 mkdocs-redirects==1.2.2
+ pip install mkdocs==1.6.1 mkdocs-material==9.6.14 mkdocs-glightbox==0.4.0 mkdocs-meta-manager==1.1.0 mkdocs-redirects==1.2.2 "mkdocs-llmstxt-md>=0.2.0"
- name: Check documentation build
run: mkdocs build --strict
diff --git a/.pipelex/inference/backends/pipelex_gateway_models.md b/.pipelex/inference/backends/pipelex_gateway_models.md
index 2e5ad26a0..5d9f7e541 100644
--- a/.pipelex/inference/backends/pipelex_gateway_models.md
+++ b/.pipelex/inference/backends/pipelex_gateway_models.md
@@ -272,7 +272,7 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td>gpt-5.3-codex</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
@@ -453,12 +453,13 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<thead>
<tr>
<th rowspan="2">Model</th>
-<th colspan="2" style="text-align:center;background-color:rgba(33,150,243,0.15)">Inputs</th>
+<th colspan="3" style="text-align:center;background-color:rgba(33,150,243,0.15)">Inputs</th>
<th colspan="2" style="text-align:center;background-color:rgba(76,175,80,0.15)">Outputs</th>
</tr>
<tr>
<th style="text-align:center;background-color:rgba(33,150,243,0.15)">image</th>
<th style="text-align:center;background-color:rgba(33,150,243,0.15)">pdf</th>
+<th style="text-align:center;background-color:rgba(33,150,243,0.15)">web_page</th>
<th style="text-align:center;background-color:rgba(76,175,80,0.15)">pages</th>
<th style="text-align:center;background-color:rgba(76,175,80,0.15)">captions</th>
</tr>
@@ -468,6 +469,7 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td>azure-document-intelligence</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
@@ -475,12 +477,14 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td>deepseek-ocr</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
</tr>
<tr>
-<td>mistral-document-ai-2505</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td>linkup-fetch</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
@@ -538,6 +542,12 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
<tr>
+<td>nano-banana-2</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+</tr>
+<tr>
<td>nano-banana-pro</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
@@ -548,6 +558,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-18T17:53:18Z
+> Last updated: 2026-03-19T01:52:02Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/.pipelex/inference/backends/pipelex_gateway_models_plain.md b/.pipelex/inference/backends/pipelex_gateway_models_plain.md
index 6ff1871dd..1acda4419 100644
--- a/.pipelex/inference/backends/pipelex_gateway_models_plain.md
+++ b/.pipelex/inference/backends/pipelex_gateway_models_plain.md
@@ -101,7 +101,7 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- inputs: text, images, pdf
- outputs: text, structured
- **gpt-5.3-codex**
- - inputs: text, images
+ - inputs: text, images, pdf
- outputs: text, structured
- **gpt-5.4**
- inputs: text, images
@@ -175,8 +175,8 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **deepseek-ocr**
- inputs: image
- outputs: pages
-- **mistral-document-ai-2505**
- - inputs: image, pdf
+- **linkup-fetch**
+ - inputs: web_page
- outputs: pages
@@ -199,12 +199,15 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **nano-banana**
- inputs: text, images
- outputs: image
+- **nano-banana-2**
+ - inputs: text, images
+ - outputs: image
- **nano-banana-pro**
- inputs: text, images
- outputs: image
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-18T17:53:18Z
+> Last updated: 2026-03-19T01:52:02Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 40b498810..ff9c75b18 100644
--- a/Makefile
+++ b/Makefile
@@ -806,49 +806,23 @@ SITE_DOMAIN := $(shell cat docs/CNAME 2>/dev/null | tr -d '[:space:]')
# Deployed by docs-deploy-root. Note: docs/robots.txt is a no-op — it only
# lands at /latest/robots.txt, which crawlers ignore per RFC 9309.
#
-# IMPORTANT: `Disallow: /` blocks ALL root paths not explicitly `Allow`ed —
-# including root-level files like /sitemap.xml. The `Sitemap:` directive tells
-# crawlers *where* the sitemap is, but does NOT override Disallow rules —
-# crawlers still need permission to fetch the URL. Any root-level file that
-# crawlers must access needs its own `Allow:` line.
+# Targeted Disallow: block versioned paths and non-content pages, allow
+# everything else (root-level files like /sitemap.xml, /llms.txt are
+# implicitly allowed).
define ROOT_ROBOTS_TXT
User-agent: *
Allow: /latest/
Allow: /sitemap.xml
-Disallow: /
+Allow: /llms.txt
+Allow: /llms-full.txt
+Disallow: /0.
+Disallow: /pre-release/
+Disallow: /404.html
+
Sitemap: https://$(SITE_DOMAIN)/sitemap.xml
endef
export ROOT_ROBOTS_TXT
-define ROOT_INDEX_HTML
-<!DOCTYPE html>
-<html>
-<head>
- <meta charset="utf-8">
- <title>Redirecting to latest documentation...</title>
- <meta http-equiv="refresh" content="0;url=/latest/">
- <link rel="canonical" href="https://$(SITE_DOMAIN)/latest/">
- <style>
- body {
- margin: 0;
- min-height: 100vh;
- display: flex;
- align-items: center;
- justify-content: center;
- background: #2e303e;
- color: #ccc;
- font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
- }
- a { color: #45BF9F; text-decoration: none; }
- </style>
-</head>
-<body>
- <p>Redirecting to <a href="/latest/">latest documentation</a>&#8230;</p>
-</body>
-</html>
-endef
-export ROOT_INDEX_HTML
-
docs: env
$(call PRINT_TITLE,"Serving documentation with mkdocs")
$(VENV_MKDOCS) serve -a 127.0.0.1:8000 -f "$(CURDIR)/mkdocs.yml" --watch "$(CURDIR)/docs" -s
@@ -905,11 +879,15 @@ endif
git worktree add "$$TMPDIR" gh-pages && \
cp docs/404.html "$$TMPDIR/404.html" && \
echo "$$ROOT_ROBOTS_TXT" > "$$TMPDIR/robots.txt" && \
- echo "$$ROOT_INDEX_HTML" > "$$TMPDIR/index.html" && \
+ sed 's/__SITE_DOMAIN__/$(SITE_DOMAIN)/g' docs/root-index.html > "$$TMPDIR/index.html" && \
sed 's|/[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*[^/]*/|/latest/|g' "$$TMPDIR/latest/sitemap.xml" > "$$TMPDIR/sitemap.xml" && \
+ if [ -f "$$TMPDIR/latest/llms.txt" ]; then cp "$$TMPDIR/latest/llms.txt" "$$TMPDIR/llms.txt"; fi && \
+ if [ -f "$$TMPDIR/latest/llms-full.txt" ]; then cp "$$TMPDIR/latest/llms-full.txt" "$$TMPDIR/llms-full.txt"; fi && \
cd "$$TMPDIR" && \
git add 404.html robots.txt index.html sitemap.xml && \
- (git diff --cached --quiet || git commit -m "Update root assets (404.html, robots.txt, index.html, sitemap.xml)") && \
+ if [ -f llms.txt ]; then git add llms.txt; fi && \
+ if [ -f llms-full.txt ]; then git add llms-full.txt; fi && \
+ (git diff --cached --quiet || git commit -m "Update root assets (404.html, robots.txt, index.html, sitemap.xml, llms.txt)") && \
git push origin gh-pages
docs-delete: export PATH := $(VIRTUAL_ENV)/bin:$(PATH)
diff --git a/docs/root-index.html b/docs/root-index.html
new file mode 100644
index 000000000..94d702522
--- /dev/null
+++ b/docs/root-index.html
@@ -0,0 +1,72 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta http-equiv="refresh" content="0;url=/latest/">
+<link rel="canonical" href="https://__SITE_DOMAIN__/latest/">
+<title>Pipelex — Build & Run AI Methods</title>
+<meta name="description" content="Pipelex is an open-source library and CLI for building and running executable AI methods. Methods are reusable, typed AI procedures declared in MTHDS and executed across 60+ models.">
+<meta property="og:title" content="Pipelex — Build & Run AI Methods">
+<meta property="og:description" content="Pipelex is an open-source library and CLI for building and running executable AI methods. Methods are reusable, typed AI procedures declared in MTHDS and executed across 60+ models.">
+<meta property="og:url" content="https://__SITE_DOMAIN__/latest/">
+<meta property="og:type" content="website">
+<style>
+ body {
+ margin: 0;
+ padding: 2rem;
+ background: #1a1a1a;
+ color: #d4d4d4;
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+ line-height: 1.6;
+ max-width: 720px;
+ margin: 0 auto;
+ }
+ a { color: #45BF9F; }
+ h1 { color: #e5e5e5; }
+ ul { padding-left: 1.2rem; }
+ .redirect-notice { color: #888; font-size: 0.85rem; margin-top: 2rem; }
+</style>
+</head>
+<body>
+<h1>Pipelex: Build & Run AI Methods</h1>
+<p>A method is a reusable, typed AI procedure declared in a .mthds file and executed by Pipelex. Each step is explicit, each output is structured, and every run is repeatable.</p>
+<h2>What a Method Looks Like</h2>
+<pre><code>[pipe.summarize_article]
+type = "PipeLLM"
+inputs = { article = "Text", audience = "Text" }
+output = "Text"
+prompt = "Summarize $article in three bullet points for $audience."
+</code></pre>
+<p>From here, Pipelex handles model routing across 60+ models, structured output parsing, and pipeline orchestration.</p>
+<h2>Why Methods?</h2>
+<ul>
+<li><strong>Declarative</strong> — Express business logic in human-readable .mthds files that work across models.</li>
+<li><strong>Typed</strong> — Concepts are semantic types: AI understands what you mean, and every input and output connects with purpose.</li>
+<li><strong>Repeatable</strong> — Deterministic orchestration that leaves exactly the room you want for AI to express its intelligence.</li>
+<li><strong>Composable</strong> — Chain pipes into sequences, nest methods inside methods, and share them with the community.</li>
+</ul>
+<h2>Capabilities</h2>
+<ul>
+<li><a href="/latest/features/concepts/">Typed Concepts</a> — Semantic types that give meaning to every input and output.</li>
+<li><a href="/latest/features/pipe-operators/">Pipe Operators</a> — LLM calls, document extraction, image generation, web search, composition, and custom functions.</li>
+<li><a href="/latest/features/pipeline-orchestration/">Pipeline Orchestration</a> — Sequence, parallel, batch, and conditional controllers.</li>
+<li><a href="/latest/features/llm-integration/">60+ AI Models</a> — OpenAI, Anthropic, Mistral, Google, Deepseek, Hugging Face, and more.</li>
+<li><a href="/latest/features/validation-dry-run/">Validation and Dry Run</a> — Catch errors without spending tokens.</li>
+<li><a href="/latest/features/cli/">CLI and Tooling</a> — Init, build, validate, run, and graph visualization.</li>
+</ul>
+<h2>Get Started</h2>
+<ul>
+<li><a href="/latest/get-started/build-with-claude-code/">Build with Claude Code</a> — Describe what you want in natural language.</li>
+<li><a href="/latest/get-started/mthds-language-tutorial/">MTHDS Language Tutorial</a> — Learn concepts, pipes, sequences, and structured outputs.</li>
+<li><a href="/latest/cookbook/">Cookbook Examples</a> — From Hello World to document extraction and image generation.</li>
+</ul>
+<h2>The MTHDS Ecosystem</h2>
+<ul>
+<li><a href="https://mthds.ai">mthds.ai</a> — The MTHDS language specification</li>
+<li><a href="https://mthds.sh">mthds.sh</a> — The Methods Hub for discovering and sharing methods</li>
+<li><a href="https://github.com/Pipelex/pipelex">GitHub</a> — Pipelex source code</li>
+</ul>
+<p>For AI agents: see <a href="/llms.txt">/llms.txt</a> for a machine-readable index of this documentation.</p>
+<p class="redirect-notice">Redirecting to <a href="/latest/">Pipelex Documentation</a>&#8230;</p>
+</body>
+</html>
diff --git a/mkdocs.yml b/mkdocs.yml
index b7e0e2cfb..55b6e4af8 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -177,6 +177,136 @@ plugins:
draggable: true
auto_caption: false
caption_position: bottom
+ - llmstxt-md:
+ sections:
+ Get Started:
+ - index.md: "Build & Run AI Methods (landing page)"
+ - get-started/build-with-claude-code.md: "Build with Claude Code"
+ - get-started/mthds-language-tutorial.md: "The MTHDS Language Tutorial"
+ - get-started/configure-ai-providers.md: "Configure AI Providers"
+ Cookbook Examples:
+ - cookbook/index.md: "Cookbook Overview"
+ - cookbook/hello-world.md: "Hello World"
+ - cookbook/summarize.md: "Summarize"
+ - cookbook/extract-generic.md: "Extract Generic Document"
+ - cookbook/extract-invoice.md: "Extract Invoice"
+ - cookbook/extract-proof-of-purchase.md: "Extract Proof of Purchase"
+ - cookbook/extract-dpe.md: "Extract DPE (Energy Diagnostic)"
+ - cookbook/extract-table.md: "Extract Table from Image"
+ - cookbook/extract-gantt.md: "Extract Gantt Chart"
+ - cookbook/extract-markdown.md: "Extract Markdown from Document"
+ - cookbook/extract-slides.md: "Extract Slides from Presentation"
+ - cookbook/design-slides.md: "Design Slides"
+ - cookbook/generate-image.md: "Generate Image"
+ - cookbook/gen-synthetic-data.md: "Generate Synthetic Data"
+ - cookbook/gen-expense-data.md: "Generate Expense Data with Receipts"
+ - cookbook/using-inference-plugins.md: "Using Inference Plugins"
+ Viewpoint:
+ - viewpoint/viewpoint.md: "The Know-How Graph"
+ Features:
+ - features/index.md: "Features Overview"
+ - features/mthds-language.md: "MTHDS Language"
+ - features/concepts.md: "Concepts & Structured Types"
+ - features/pipe-operators.md: "Pipe Operators"
+ - features/pipeline-orchestration.md: "Pipeline Orchestration"
+ - features/gateway.md: "Pipelex Gateway & Model Access"
+ - features/llm-integration.md: "LLM Integration"
+ - features/document-extraction.md: "Document Extraction"
+ - features/image-generation.md: "Image Generation"
+ - features/web-search.md: "Web Search"
+ - features/claude-code-skills-plugin.md: "Claude Code Skills Plugin"
+ - features/cli.md: "CLI"
+ - features/plxt.md: "plxt Formatter & Linter"
+ - features/execution-graph.md: "Execution Graph Visualization"
+ - features/validation-dry-run.md: "Validation & Dry Run"
+ - features/telemetry.md: "Telemetry & Observability"
+ - features/cloud-storage.md: "Cloud Storage"
+ - features/cost-tracking.md: "Cost Tracking & Reporting"
+ - features/configuration.md: "Configuration System"
+ - features/advanced-customizations.md: "Advanced Customizations"
+ Building Methods:
+ - building-methods/kick-off-a-methods-project.md: "Kick off a Method Project"
+ - building-methods/pipelex-bundle-specification.md: "Pipelex Bundle Specification"
+ - building-methods/domain.md: "Domain"
+ - building-methods/concepts/define_your_concepts.md: "Define Your Concepts"
+ - building-methods/concepts/inline-structures.md: "Inline Structures"
+ - building-methods/concepts/python-classes.md: "Python Classes"
+ - building-methods/concepts/native-concepts.md: "Native Concepts"
+ - building-methods/concepts/refining-concepts.md: "Refining Concepts"
+ - building-methods/pipes/index.md: "Design and Run Methods Overview"
+ - building-methods/libraries.md: "Libraries"
+ - building-methods/packages.md: "Packages"
+ - building-methods/pipes/executing-pipelines.md: "Executing Pipelines"
+ - building-methods/pipes/provide-inputs.md: "Providing Inputs to Pipelines"
+ - building-methods/pipes/working-memory.md: "Working Memory"
+ - building-methods/pipes/pipe-output.md: "Pipe Output"
+ - building-methods/pipes/understanding-multiplicity.md: "Understanding Multiplicity"
+ - building-methods/pipes/pipe-operators/index.md: "Pipe Operators Overview"
+ - building-methods/pipes/pipe-operators/PipeLLM.md: "PipeLLM"
+ - building-methods/pipes/pipe-operators/PipeExtract.md: "PipeExtract"
+ - building-methods/pipes/pipe-operators/PipeImgGen.md: "PipeImgGen"
+ - building-methods/pipes/pipe-operators/PipeSearch.md: "PipeSearch"
+ - building-methods/pipes/pipe-operators/PipeCompose.md: "PipeCompose"
+ - building-methods/pipes/pipe-operators/PipeFunc.md: "PipeFunc"
+ - building-methods/pipes/pipe-controllers/index.md: "Pipe Controllers Overview"
+ - building-methods/pipes/pipe-controllers/PipeSequence.md: "PipeSequence"
+ - building-methods/pipes/pipe-controllers/PipeParallel.md: "PipeParallel"
+ - building-methods/pipes/pipe-controllers/PipeBatch.md: "PipeBatch"
+ - building-methods/pipes/pipe-controllers/PipeCondition.md: "PipeCondition"
+ - building-methods/configure-ai-llm-to-optimize-methods.md: "Optimize Cost & Quality"
+ - building-methods/llm-structured-generation-config.md: "LLM Structured Generation"
+ - building-methods/adapt-to-llm-prompting-style-openai-anthropic-mistral.md: "LLM Prompting Style"
+ Reference:
+ - tools/cli/index.md: "CLI Reference Overview"
+ - tools/cli/init.md: "CLI Init"
+ - tools/cli/validate.md: "CLI Validate"
+ - tools/cli/run.md: "CLI Run"
+ - tools/cli/show.md: "CLI Show"
+ - tools/cli/pkg.md: "CLI Pkg"
+ - tools/cli/build/index.md: "CLI Build Overview"
+ - tools/cli/build/runner.md: "CLI Build Runner"
+ - tools/cli/build/structures.md: "CLI Build Structures"
+ - tools/cli/build/inputs.md: "CLI Build Inputs"
+ - tools/cli/build/output.md: "CLI Build Output"
+ - tools/cli/agent-cli.md: "Agent CLI"
+ - configuration/index.md: "Configuration Overview"
+ - configuration/config-pipeline-validation/dry-run-config.md: "Dry Run Configuration"
+ - configuration/config-practical/logging-config.md: "Logging Configuration"
+ - configuration/config-practical/pipe-run-config.md: "Pipe Run Configuration"
+ - configuration/config-practical/reporting-config.md: "Reporting Configuration"
+ - configuration/config-practical/telemetry-config.md: "Telemetry Configuration"
+ - configuration/config-technical/aws-config.md: "AWS Configuration"
+ - configuration/config-technical/cogt-config.md: "Cogt Configuration"
+ - configuration/config-technical/inference-backend-config.md: "LLM Providers & Models Configuration"
+ - configuration/config-technical/library-config.md: "Library Configuration"
+ - configuration/config-advanced/feature-config.md: "Feature Configuration"
+ - tools/plxt.md: "plxt (Formatter & Linter)"
+ - tools/logging.md: "Logging"
+ - analytics/data-extraction.md: "Observer Data Extraction"
+ - setup/telemetry.md: "Telemetry"
+ - setup/gateway-models.md: "Gateway Available Models"
+ Advanced:
+ - advanced/index.md: "Advanced Customizations Overview"
+ - advanced/secrets-provider-injection.md: "Secrets Provider"
+ - advanced/reporting-delegate-injection.md: "Reporting Delegate"
+ - advanced/storage-provider-injection.md: "Storage Provider"
+ - advanced/observer-provider-injection.md: "Observer"
+ - advanced/content-generator-injection.md: "Content Generator"
+ - advanced/pipe-router-injection.md: "Pipe Router"
+ Under the Hood:
+ - under-the-hood/index.md: "Under the Hood Overview"
+ - under-the-hood/architecture-overview.md: "Architecture Overview"
+ - under-the-hood/dry-run-mock-generation.md: "Dry Run Mock Generation"
+ - under-the-hood/execution-graph-tracing.md: "Execution Graph Tracing"
+ - under-the-hood/image-handling-in-llm-prompts.md: "Image Handling in LLM Prompts"
+ - under-the-hood/reasoning-controls.md: "Reasoning Controls"
+ - under-the-hood/stuffartefact-and-image-rendering.md: "StuffArtefact & Image Rendering"
+ - under-the-hood/test-profile-configuration.md: "Test Profile Configuration"
+ - under-the-hood/init-cli-flows.md: "Init CLI Flows"
+ Project:
+ - contributing.md: "Contributing"
+ - contribute/configuration-defaults-and-overrides.md: "Configuration Internals"
+ - changelog.md: "Changelog"
markdown_extensions:
- meta
diff --git a/pipelex/builder/pipe/pipe_extract_spec.py b/pipelex/builder/pipe/pipe_extract_spec.py
index f67743cc7..e37317670 100644
--- a/pipelex/builder/pipe/pipe_extract_spec.py
+++ b/pipelex/builder/pipe/pipe_extract_spec.py
@@ -45,10 +45,7 @@ class PipeExtractSpec(PipeSpec):
max_page_images: int | None = Field(
default=None, description="Max number of images to extract from pages: None=unlimited, 0=no images, N=limit to N images."
)
- page_image_captions: bool | None = Field(default=None, description="Whether to generate captions for detected images using AI.")
page_views: bool | None = Field(default=None, description="Whether to include rendered page views in the output.")
- render_js: bool | None = Field(default=None, description="Whether to render JavaScript before fetching (web extraction only).")
- include_raw_html: bool | None = Field(default=None, description="Whether to include raw HTML in the response (web extraction only).")
@override
@field_validator("output", mode="before")
@@ -93,8 +90,6 @@ class PipeExtractSpec(PipeSpec):
# Add optional extraction settings if they are set
if self.max_page_images is not None:
extract_group.renderables.append(Text.from_markup(f"Max Page Images: [bold magenta]{self.max_page_images}[/bold magenta]"))
- if self.page_image_captions is not None:
- extract_group.renderables.append(Text.from_markup(f"Generate Image Captions: [bold magenta]{self.page_image_captions}[/bold magenta]"))
if self.page_views is not None:
extract_group.renderables.append(Text.from_markup(f"Include Page Views: [bold magenta]{self.page_views}[/bold magenta]"))
@@ -115,9 +110,9 @@ class PipeExtractSpec(PipeSpec):
output=base_blueprint.output,
model=extract_model_choice,
max_page_images=self.max_page_images,
- page_image_captions=self.page_image_captions,
+ page_image_captions=None,
page_views=self.page_views,
page_views_dpi=None,
- render_js=self.render_js,
- include_raw_html=self.include_raw_html,
+ render_js=None,
+ include_raw_html=None,
)
diff --git a/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md b/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
index 2e5ad26a0..5d9f7e541 100644
--- a/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
+++ b/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
@@ -272,7 +272,7 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td>gpt-5.3-codex</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
@@ -453,12 +453,13 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<thead>
<tr>
<th rowspan="2">Model</th>
-<th colspan="2" style="text-align:center;background-color:rgba(33,150,243,0.15)">Inputs</th>
+<th colspan="3" style="text-align:center;background-color:rgba(33,150,243,0.15)">Inputs</th>
<th colspan="2" style="text-align:center;background-color:rgba(76,175,80,0.15)">Outputs</th>
</tr>
<tr>
<th style="text-align:center;background-color:rgba(33,150,243,0.15)">image</th>
<th style="text-align:center;background-color:rgba(33,150,243,0.15)">pdf</th>
+<th style="text-align:center;background-color:rgba(33,150,243,0.15)">web_page</th>
<th style="text-align:center;background-color:rgba(76,175,80,0.15)">pages</th>
<th style="text-align:center;background-color:rgba(76,175,80,0.15)">captions</th>
</tr>
@@ -468,6 +469,7 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td>azure-document-intelligence</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
@@ -475,12 +477,14 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td>deepseek-ocr</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
</tr>
<tr>
-<td>mistral-document-ai-2505</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td>linkup-fetch</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
@@ -538,6 +542,12 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
<tr>
+<td>nano-banana-2</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+</tr>
+<tr>
<td>nano-banana-pro</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
@@ -548,6 +558,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-18T17:53:18Z
+> Last updated: 2026-03-19T01:52:02Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md b/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
index 6ff1871dd..1acda4419 100644
--- a/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
+++ b/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
@@ -101,7 +101,7 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- inputs: text, images, pdf
- outputs: text, structured
- **gpt-5.3-codex**
- - inputs: text, images
+ - inputs: text, images, pdf
- outputs: text, structured
- **gpt-5.4**
- inputs: text, images
@@ -175,8 +175,8 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **deepseek-ocr**
- inputs: image
- outputs: pages
-- **mistral-document-ai-2505**
- - inputs: image, pdf
+- **linkup-fetch**
+ - inputs: web_page
- outputs: pages
@@ -199,12 +199,15 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **nano-banana**
- inputs: text, images
- outputs: image
+- **nano-banana-2**
+ - inputs: text, images
+ - outputs: image
- **nano-banana-pro**
- inputs: text, images
- outputs: image
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-18T17:53:18Z
+> Last updated: 2026-03-19T01:52:02Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/pipelex/plugins/linkup/linkup_extract_worker.py b/pipelex/plugins/linkup/linkup_extract_worker.py
index 911a1bd11..78439dbd6 100644
--- a/pipelex/plugins/linkup/linkup_extract_worker.py
+++ b/pipelex/plugins/linkup/linkup_extract_worker.py
@@ -39,7 +39,7 @@ class LinkupExtractWorker(ExtractWorkerAbstract):
raise ValueError(msg)
job_params = extract_job.job_params
- extract_images = (job_params.max_nb_images or 0) != 0
+ extract_images = job_params.max_nb_images is None or job_params.max_nb_images > 0
response: LinkupFetchResponse = await self._linkup_client.async_fetch(
url=document_uri,
diff --git a/pipelex/plugins/linkup/linkup_search_worker.py b/pipelex/plugins/linkup/linkup_search_worker.py
index 5554fe6fe..438b8f575 100644
--- a/pipelex/plugins/linkup/linkup_search_worker.py
+++ b/pipelex/plugins/linkup/linkup_search_worker.py
@@ -38,7 +38,7 @@ class LinkupSearchWorker(SearchWorkerAbstract):
job_params = search_job.job_params
search_setting = job_params.search_setting
- depth_value = SearchDepth(self.inference_model.model_id)
+ depth_value = SearchDepth(self.inference_model.model_id.rsplit("/", 1)[-1])
response: LinkupSourcedAnswer = await self._linkup_client.async_search(
query=search_job.query,
depth=depth_value, # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
@@ -80,7 +80,7 @@ class LinkupSearchWorker(SearchWorkerAbstract):
job_params = search_job.job_params
search_setting = job_params.search_setting
- depth_value = SearchDepth(self.inference_model.model_id)
+ depth_value = SearchDepth(self.inference_model.model_id.rsplit("/", 1)[-1])
response = await self._linkup_client.async_search(
query=search_job.query,
depth=depth_value, # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
diff --git a/pyproject.toml b/pyproject.toml
index 4eef47836..a33c719ec 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -74,12 +74,13 @@ linkup = ["linkup-sdk>=0.12.0"]
mistralai = ["mistralai>=1.12.0"]
s3 = ["boto3>=1.34.131", "aioboto3>=13.4.0"]
docs = [
+ "mike>=2.1.3",
"mkdocs>=1.6.1",
"mkdocs-glightbox>=0.4.0",
"mkdocs-material>=9.6.14",
"mkdocs-meta-manager>=1.1.0",
- "mike>=2.1.3",
"mkdocs-redirects==1.2.2",
+ "mkdocs-llmstxt-md>=0.2.0",
]
dev = [
diff --git a/tests/integration/pipelex/cogt/test_extract.py b/tests/integration/pipelex/cogt/test_extract.py
index 58f741915..193ee9c17 100644
--- a/tests/integration/pipelex/cogt/test_extract.py
+++ b/tests/integration/pipelex/cogt/test_extract.py
@@ -152,7 +152,7 @@ class TestExtract:
)
extract_output = await extract_worker.extract_pages(extract_job=extract_job)
assert extract_output.pages
- page = extract_output.pages[0]
+ page = next(iter(extract_output.pages.values()))
assert page.text, "Web extraction should return markdown text"
pretty_print(page.text, title="Extracted markdown")
if page.raw_html:
diff --git a/uv.lock b/uv.lock
index de0cf98b1..9e0f984f7 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2262,6 +2262,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl", hash = "sha256:23a431ea802b60b1030c73323db2eed6ba859df1a0822ce575afa43e0ea3f47e", size = 26458, upload-time = "2025-10-23T14:55:17.43Z" },
]
+[[package]]
+name = "mkdocs-llmstxt-md"
+version = "0.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "mkdocs" },
+ { name = "ruff" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/be/86/a899ef021e44e358544b31fd3e34aa566ff876650e7dbede3e677053a492/mkdocs_llmstxt_md-0.2.0.tar.gz", hash = "sha256:c42c86fc316c5e96920b403abfffda7615fe519baca9cdb716e9ff77b703570f", size = 8780, upload-time = "2025-08-01T14:14:10.926Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/48/96/519526b02a4429262b0e8728765a0bda8a987d1fd4e6ac7655f26ce2a245/mkdocs_llmstxt_md-0.2.0-py3-none-any.whl", hash = "sha256:ce836a6e7bc7f7d0987c8ac0703f04b51f1d7d0096a5700b042161fbbaea5175", size = 7551, upload-time = "2025-08-01T14:14:09.558Z" },
+]
+
[[package]]
name = "mkdocs-material"
version = "9.7.5"
@@ -3017,9 +3030,9 @@ name = "ocrmac"
version = "1.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "click" },
- { name = "pillow" },
- { name = "pyobjc-framework-vision" },
+ { name = "click", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+ { name = "pillow", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+ { name = "pyobjc-framework-vision", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/5e/07/3e15ab404f75875c5e48c47163300eb90b7409044d8711fc3aaf52503f2e/ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9", size = 1454317, upload-time = "2026-01-08T16:44:26.412Z" }
wheels = [
@@ -3509,6 +3522,7 @@ docs = [
{ name = "mike" },
{ name = "mkdocs" },
{ name = "mkdocs-glightbox" },
+ { name = "mkdocs-llmstxt-md" },
{ name = "mkdocs-material" },
{ name = "mkdocs-meta-manager" },
{ name = "mkdocs-redirects" },
@@ -3569,6 +3583,7 @@ requires-dist = [
{ name = "mistralai", marker = "extra == 'mistralai'", specifier = ">=1.12.0" },
{ name = "mkdocs", marker = "extra == 'docs'", specifier = ">=1.6.1" },
{ name = "mkdocs-glightbox", marker = "extra == 'docs'", specifier = ">=0.4.0" },
+ { name = "mkdocs-llmstxt-md", marker = "extra == 'docs'", specifier = ">=0.2.0" },
{ name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.6.14" },
{ name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = ">=1.1.0" },
{ name = "mkdocs-redirects", marker = "extra == 'docs'", specifier = "==1.2.2" },
@@ -4164,7 +4179,7 @@ name = "pyobjc-framework-cocoa"
version = "12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "pyobjc-core" },
+ { name = "pyobjc-core", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/02/a3/16ca9a15e77c061a9250afbae2eae26f2e1579eb8ca9462ae2d2c71e1169/pyobjc_framework_cocoa-12.1.tar.gz", hash = "sha256:5556c87db95711b985d5efdaaf01c917ddd41d148b1e52a0c66b1a2e2c5c1640", size = 2772191, upload-time = "2025-11-14T10:13:02.069Z" }
wheels = [
@@ -4182,8 +4197,8 @@ name = "pyobjc-framework-coreml"
version = "12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "pyobjc-core" },
- { name = "pyobjc-framework-cocoa" },
+ { name = "pyobjc-core", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+ { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/30/2d/baa9ea02cbb1c200683cb7273b69b4bee5070e86f2060b77e6a27c2a9d7e/pyobjc_framework_coreml-12.1.tar.gz", hash = "sha256:0d1a4216891a18775c9e0170d908714c18e4f53f9dc79fb0f5263b2aa81609ba", size = 40465, upload-time = "2025-11-14T10:14:02.265Z" }
wheels = [
@@ -4201,8 +4216,8 @@ name = "pyobjc-framework-quartz"
version = "12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "pyobjc-core" },
- { name = "pyobjc-framework-cocoa" },
+ { name = "pyobjc-core", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+ { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/94/18/cc59f3d4355c9456fc945eae7fe8797003c4da99212dd531ad1b0de8a0c6/pyobjc_framework_quartz-12.1.tar.gz", hash = "sha256:27f782f3513ac88ec9b6c82d9767eef95a5cf4175ce88a1e5a65875fee799608", size = 3159099, upload-time = "2025-11-14T10:21:24.31Z" }
wheels = [
@@ -4220,10 +4235,10 @@ name = "pyobjc-framework-vision"
version = "12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "pyobjc-core" },
- { name = "pyobjc-framework-cocoa" },
- { name = "pyobjc-framework-coreml" },
- { name = "pyobjc-framework-quartz" },
+ { name = "pyobjc-core", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+ { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+ { name = "pyobjc-framework-coreml", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+ { name = "pyobjc-framework-quartz", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c2/5a/08bb3e278f870443d226c141af14205ff41c0274da1e053b72b11dfc9fb2/pyobjc_framework_vision-12.1.tar.gz", hash = "sha256:a30959100e85dcede3a786c544e621ad6eb65ff6abf85721f805822b8c5fe9b0", size = 59538, upload-time = "2025-11-14T10:23:21.979Z" }
wheels = [