Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Introduce experimental host protocol overlays and migrate Analysis (#11416)

Trevor Manzcb8825f6e961
diff --git a/.gitattributes b/.gitattributes
index b13de8ba44..2d958382ca 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2,3 +2,6 @@
pixi.lock merge=binary linguist-language=YAML linguist-generated=true
packages/client/src/protocol.ts linguist-generated=true
packages/client/src/operations.ts linguist-generated=true
+
+# SSE frames end with a blank line, including the final frame.
+protocols/host/**/*.sse whitespace=-blank-at-eof
diff --git a/.github/actions/build-frontend/action.yaml b/.github/actions/build-frontend/action.yaml
index 3ad025fc89..697276407a 100644
--- a/.github/actions/build-frontend/action.yaml
+++ b/.github/actions/build-frontend/action.yaml
@@ -27,6 +27,9 @@ runs:
cache: pnpm
cache-dependency-path: '**/pnpm-lock.yaml'
+ - name: Set up uv for host protocol generation
+ uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+
- name: 📦 Build frontend
shell: bash
run: make fe
diff --git a/.github/actions/install/action.yaml b/.github/actions/install/action.yaml
index 7924515330..e441a258f8 100644
--- a/.github/actions/install/action.yaml
+++ b/.github/actions/install/action.yaml
@@ -18,3 +18,6 @@ runs:
- name: 📥 Install dependencies
shell: bash
run: pnpm install --ignore-scripts --frozen-lockfile
+
+ - name: Set up uv for host protocol generation
+ uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
diff --git a/.github/workflows/test_host_protocol.yaml b/.github/workflows/test_host_protocol.yaml
index 8c39f6a53c..9842fdc86c 100644
--- a/.github/workflows/test_host_protocol.yaml
+++ b/.github/workflows/test_host_protocol.yaml
@@ -7,6 +7,7 @@ on:
- "marimo/_core/protocol.py"
- "tests/_core/test_protocol.py"
- "tests/_lint/test_host_protocol_codegen.py"
+ - "tests/_lint/test_host_protocol_overlays.py"
- "pyproject.toml"
- "Makefile"
- ".github/workflows/test_host_protocol.yaml"
@@ -28,7 +29,7 @@ jobs:
- name: Check Python model generation and portable fixtures
run: make host-protocol-check
- name: Check generated Python bindings against fixtures
- run: uv run --group test pytest tests/_core/test_protocol.py tests/_lint/test_host_protocol_codegen.py
+ run: uv run --group test pytest tests/_core/test_protocol.py tests/_lint/test_host_protocol_codegen.py tests/_lint/test_host_protocol_overlays.py
- name: Lint protocol tooling
run: |
uv run ruff check protocols
diff --git a/Makefile b/Makefile
index fb06a58543..2e0f825534 100644
--- a/Makefile
+++ b/Makefile
@@ -128,13 +128,15 @@ fe-codegen:
.PHONY: host-protocol-codegen
# Generate host protocol Python bindings
host-protocol-codegen:
- uv run --locked --script protocols/host/generate.py
+ uv run --no-config --locked --script protocols/host/generate.py --include-experimental
.PHONY: host-protocol-check
# Check host model generation and portable protocol fixtures
host-protocol-check:
- uv run --locked --script protocols/host/generate.py --check
- uv run --locked --project protocols/host/conformance pytest protocols/host/conformance
+ @tmp=$$(mktemp -d); trap 'rm -rf "$$tmp"' EXIT; \
+ uv run --no-config --locked --script protocols/host/generate.py --output-dir "$$tmp"
+ uv run --no-config --locked --script protocols/host/generate.py --include-experimental --check
+ uv run --locked --project protocols/host/conformance pytest -c protocols/host/conformance/pyproject.toml protocols/host/conformance protocols/host/experimental
.PHONY: fe-plugin-schema
# 🔄 Generate frontend plugin API schema
diff --git a/marimo/_core/protocol.py b/marimo/_core/protocol.py
index ba2bc97dca..0b5b692a1a 100644
--- a/marimo/_core/protocol.py
+++ b/marimo/_core/protocol.py
@@ -1,7 +1,8 @@
# Copyright 2026 Marimo. All rights reserved.
# AUTO-GENERATED FILE — DO NOT EDIT.
# Generated from protocols/host/openapi.yaml.
-# Regenerate with `make host-protocol-codegen`.
+# Experimental features: analysis
+# Regenerate with `uv run --no-config --locked --script protocols/host/generate.py --include-experimental=analysis`.
# ruff: noqa: TC003 - msgspec resolves annotations at runtime
# ruff: noqa: RUF036 - the generator orders unions; the wire shape is the same
@@ -260,7 +261,6 @@ ObjectType: TypeAlias = Annotated[
"project",
"notebook",
"document",
- "analysis",
"cell",
"runtime",
"attachment",
@@ -270,9 +270,10 @@ ObjectType: TypeAlias = Annotated[
"display",
"variable",
"custom",
+ "analysis",
],
Meta(
- description="The kinds a stream carries, and the values `types` accepts. The\ncatalog stream carries host, project, notebook, runtime, and\nattachment. A notebook's stream carries notebook, document,\nanalysis, cell, runtime, attachment, run, execution, console,\ndisplay, variable, and custom. Custom events describe occurrences\nrather than object state.\n"
+ description="The kinds a stream carries, and the values `types` accepts. The\ncatalog stream carries host, project, notebook, runtime, and\nattachment. A notebook's stream carries notebook, document,\ncell, runtime, attachment, run, execution, console,\ndisplay, variable, and custom. Custom events describe occurrences\nrather than object state.\n"
),
]
@@ -1132,7 +1133,7 @@ class Error(Struct, frozen=True):
kind: Annotated[
str,
Meta(
- description="The producer's identifier, scoped by source. Open; this protocol defines `exception`, `interrupted`, and `internal` for executions, and reserves `multiple-definitions` and `cycle-dependencies` for analysis, because every host schedules them the same way. Clients show unrecognized kinds as plain errors."
+ description="The producer's identifier, scoped by source. Open; this protocol defines `exception`, `interrupted`, and `internal` for executions. Clients show unrecognized kinds as plain errors."
),
]
message: Annotated[
@@ -1906,7 +1907,6 @@ Event: TypeAlias = Annotated[
| ProjectEvent
| NotebookEvent
| DocumentEvent
- | AnalysisEvent
| CellEvent
| RuntimeEvent
| AttachmentEvent
@@ -1927,7 +1927,8 @@ Event: TypeAlias = Annotated[
| DisplayRemovedEvent
| VariableRemovedEvent
| ResetEvent
- | ReadyEvent,
+ | ReadyEvent
+ | AnalysisEvent,
Meta(
description="A record, removal, or control message from a host stream.\nThe data is decoded JSON. Snapshot records and reset events omit\nthe cursor; ready and live events carry it. Unknown event names\nbelong to later versions or host extensions.\n"
),
diff --git a/marimo/_core/protocol_events.py b/marimo/_core/protocol_events.py
index 0b4f5e55ff..a1dc8ffe85 100644
--- a/marimo/_core/protocol_events.py
+++ b/marimo/_core/protocol_events.py
@@ -1,7 +1,8 @@
# Copyright 2026 Marimo. All rights reserved.
# AUTO-GENERATED FILE — DO NOT EDIT.
# Generated from protocols/host/openapi.yaml.
-# Regenerate with `make host-protocol-codegen`.
+# Experimental features: analysis
+# Regenerate with `uv run --no-config --locked --script protocols/host/generate.py --include-experimental=analysis`.
"""Typed envelope adapters. Payload ownership stays with the caller."""
from __future__ import annotations
diff --git a/packages/client/README.md b/packages/client/README.md
index b0eebfdbfb..5acf7488c8 100644
--- a/packages/client/README.md
+++ b/packages/client/README.md
@@ -133,13 +133,23 @@ pnpm --dir packages/client build --watch
After changing OpenAPI, regenerate the protocol types, schemas, and
operation definitions. `codegen:check` checks that generated files match
-the specification.
+the specification. Generation requires `uv` as well as the Node toolchain:
+both language generators use the same Python overlay composer. The checked-in
+client includes experimental features, including Analysis.
```sh
pnpm --dir packages/client codegen
pnpm --dir packages/client codegen:check
```
+The package commands opt into all experimental features so Turbo builds retain
+the APIs the handwritten client consumes. Use `codegen:stable` for stable-only
+bindings, using
+`--output-dir` to keep them separate from the client's checked-in bindings. Use
+`--include-experimental=analysis` to select a feature. This selects generated
+contracts; it does not create a separate stable build of the client package.
+See the [host protocol tooling](../../protocols/host/README.md) for details.
+
Run the package's tests and typecheck. Omit `--run` to watch tests while
developing.
diff --git a/packages/client/package.json b/packages/client/package.json
index d8916cd3e4..0fcc88f29f 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -16,10 +16,11 @@
],
"scripts": {
"build": "tsc -p tsconfig.build.json",
- "codegen": "node --experimental-strip-types scripts/generate-protocol.ts",
- "codegen:check": "node --experimental-strip-types scripts/generate-protocol.ts --check",
+ "codegen": "node --experimental-strip-types scripts/generate-protocol.ts --include-experimental",
+ "codegen:check": "node --experimental-strip-types scripts/generate-protocol.ts --include-experimental --check",
"test": "vitest",
- "typecheck": "tsgo"
+ "typecheck": "tsgo",
+ "codegen:stable": "node --experimental-strip-types scripts/generate-protocol.ts"
},
"dependencies": {
"zod": "^4.6.5"
diff --git a/packages/client/scripts/generate-protocol.ts b/packages/client/scripts/generate-protocol.ts
index d86ad1a70e..6e7bae7634 100644
--- a/packages/client/scripts/generate-protocol.ts
+++ b/packages/client/scripts/generate-protocol.ts
@@ -5,16 +5,54 @@ import NodeFs from "node:fs";
import NodeOs from "node:os";
import NodePath from "node:path";
import NodeUrl from "node:url";
+import NodeUtil from "node:util";
import { generate, type OpenApiDocument } from "orval";
-import { parse } from "yaml";
const root = NodeUrl.fileURLToPath(new URL("../../../", import.meta.url));
-const source = NodePath.join(root, "packages/client/src");
-const header = `/* Copyright 2026 Marimo. All rights reserved. */
-// Generated from protocols/host/openapi.yaml. Do not edit.
-// Regenerate with: pnpm --dir packages/client codegen
-
-`;
+const { values: args } = NodeUtil.parseArgs({
+ options: {
+ check: { type: "boolean", default: false },
+ "include-experimental": { type: "string" },
+ "output-dir": { type: "string" },
+ },
+ args: process.argv
+ .slice(2)
+ .map((arg) =>
+ arg === "--include-experimental" ? "--include-experimental=all" : arg,
+ ),
+});
+const source = args["output-dir"] ?? NodePath.join(root, "packages/client/src");
+const selection = args["include-experimental"];
+const spec: OpenApiDocument = JSON.parse(
+ NodeChildProcess.execFileSync(
+ "uv",
+ [
+ "run",
+ "--no-config",
+ "--locked",
+ "--script",
+ NodePath.join(root, "protocols/host/generate.py"),
+ "--schema",
+ ...(selection === undefined
+ ? []
+ : [`--include-experimental=${selection}`]),
+ ],
+ { encoding: "utf8", maxBuffer: 10 * 1024 * 1024 },
+ ),
+);
+const features = (spec["x-marimo-experimental"] ?? []) as string[];
+const regenerationCommand = features.length
+ ? `pnpm --dir packages/client codegen --include-experimental=${features.join(",")}`
+ : "pnpm --dir packages/client codegen:stable";
+const header = [
+ "/* Copyright 2026 Marimo. All rights reserved. */",
+ "// Generated from protocols/host/openapi.yaml. Do not edit.",
+ ...(features.length
+ ? [`// Experimental features: ${features.join(", ")}`]
+ : []),
+ `// Regenerate with: ${regenerationCommand}`,
+ "\n",
+].join("\n");
interface Parameter {
name: string;
@@ -152,12 +190,6 @@ const temporary = NodeFs.mkdtempSync(
NodePath.join(NodeOs.tmpdir(), "marimo-host-protocol-"),
);
try {
- const spec: OpenApiDocument = parse(
- NodeFs.readFileSync(
- NodePath.join(root, "protocols/host/openapi.yaml"),
- "utf8",
- ),
- );
const schemas = spec.components?.schemas;
if (!schemas) {
throw new Error("Host protocol has no component schemas");
@@ -188,19 +220,22 @@ try {
NodePath.join(root, "node_modules/.bin/oxfmt"),
["--config", NodePath.join(root, ".oxfmtrc.json"), output, table],
);
+ if (!args.check) {
+ NodeFs.mkdirSync(source, { recursive: true });
+ }
for (const [generated, name] of [
[output, "protocol.ts"],
[table, "operations.ts"],
] as const) {
const content = NodeFs.readFileSync(generated, "utf8");
const destination = NodePath.join(source, name);
- if (process.argv.includes("--check")) {
+ if (args.check) {
if (
!NodeFs.existsSync(destination) ||
NodeFs.readFileSync(destination, "utf8") !== content
) {
throw new Error(
- `${name} is stale; run pnpm --dir packages/client codegen`,
+ `${name} is stale; regenerate with the same feature selection and output directory`,
);
}
} else {
diff --git a/packages/client/src/__tests__/generation.test.ts b/packages/client/src/__tests__/generation.test.ts
new file mode 100644
index 0000000000..c481dbd381
--- /dev/null
+++ b/packages/client/src/__tests__/generation.test.ts
@@ -0,0 +1,75 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import NodeChildProcess from "node:child_process";
+import NodeFs from "node:fs";
+import NodePath from "node:path";
+import NodeUrl from "node:url";
+import { expect, it } from "vitest";
+
+it("generates a stable protocol that tolerates experimental stream events", () => {
+ const root = NodeUrl.fileURLToPath(new URL("../../", import.meta.url));
+ // Keep the temporary bindings in the package so Node can resolve zod.
+ const output = NodeFs.mkdtempSync(NodePath.join(root, ".protocol-test-"));
+ try {
+ const command = [
+ "--experimental-strip-types",
+ NodePath.join(root, "scripts/generate-protocol.ts"),
+ "--output-dir",
+ output,
+ ];
+ NodeChildProcess.execFileSync("node", command, { stdio: "pipe" });
+ NodeChildProcess.execFileSync("node", [...command, "--check"], {
+ stdio: "pipe",
+ });
+ NodeFs.mkdirSync(NodePath.join(output, "transport"));
+ for (const name of ["frame.ts", "stream.ts"]) {
+ NodeFs.copyFileSync(
+ NodePath.join(root, "src/transport", name),
+ NodePath.join(output, "transport", name),
+ );
+ }
+ const protocol = NodeUrl.pathToFileURL(
+ NodePath.join(output, "protocol.ts"),
+ ).href;
+ const stream = NodeUrl.pathToFileURL(
+ NodePath.join(output, "transport/stream.ts"),
+ ).href;
+ const result = NodeChildProcess.execFileSync(
+ "node",
+ [
+ "--experimental-strip-types",
+ "--input-type=module",
+ "-e",
+ `
+ import assert from "node:assert/strict";
+ import * as Protocol from ${JSON.stringify(protocol)};
+ import { EventStream } from ${JSON.stringify(stream)};
+ assert.equal("Analysis" in Protocol, false);
+ assert.equal(Protocol.ObjectType.options.includes("analysis"), false);
+ const stream = new EventStream({}, Protocol.ObjectType.options);
+ const received = [];
+ stream.subscribe(event => received.push(event));
+ stream.receive({ event: "analysis", data: {} });
+ stream.receive({ event: "ready", id: "1", data: {} });
+ stream.receive({ event: "analysis", id: "2", data: {} });
+ assert.equal(stream.cursor, "2");
+ assert.equal(stream.resume(), "2");
+ stream.receive({ event: "analysis", id: "3", data: {} });
+ stream.receive({ event: "ready", id: "3", data: {} });
+ stream.receive({ event: "reset", data: {} });
+ assert.equal(stream.cursor, null);
+ assert.deepEqual(received, [
+ { event: "ready", id: "1", data: {} },
+ { event: "ready", id: "3", data: {} },
+ { event: "reset", data: {} },
+ ]);
+ console.log("stable stream verified");
+ `,
+ ],
+ { encoding: "utf8" },
+ );
+ expect(result.trim()).toBe("stable stream verified");
+ } finally {
+ NodeFs.rmSync(output, { recursive: true, force: true });
+ }
+}, 30_000);
diff --git a/packages/client/src/__tests__/protocol.test.ts b/packages/client/src/__tests__/protocol.test.ts
index c6dd16d04d..fddd8f9a3e 100644
--- a/packages/client/src/__tests__/protocol.test.ts
+++ b/packages/client/src/__tests__/protocol.test.ts
@@ -20,8 +20,8 @@ const records = {
"record.json": Protocol.HostRecord,
"cell.json": Protocol.Cell,
"cell-withheld.json": Protocol.Cell,
- "analysis.json": Protocol.Analysis,
- "diagnostic.json": Protocol.Error,
+ "../experimental/analysis/fixtures/analysis.json": Protocol.Analysis,
+ "../experimental/analysis/fixtures/diagnostic.json": Protocol.Error,
"edit.json": Protocol.EditRequest,
"edit-conflict-request.json": Protocol.EditRequest,
"edit-response.json": Protocol.EditResponse,
@@ -45,35 +45,32 @@ describe("host protocol fixtures", () => {
},
);
- it.each(NodeFs.readdirSync(fixtures).filter((name) => name.endsWith(".sse")))(
- "parses events in %s unchanged",
- (name) => {
- const stream = NodeFs.readFileSync(
- new NodeUrl.URL(name, fixtures),
- "utf8",
- );
- let count = 0;
- for (const frame of stream.split(/\r?\n\r?\n/)) {
- const event = /^event: (.+)$/m.exec(frame)?.[1];
- if (!event && !/^data:/m.test(frame)) {
- continue;
- }
- const data = frame
- .split(/\r?\n/)
- .filter((line) => line.startsWith("data:"))
- .map((line) => line.slice(5).trimStart())
- .join("\n");
- const id = /^id: (.+)$/m.exec(frame)?.[1];
- const payload: unknown = JSON.parse(data);
- const input = { event, data: payload, ...(id ? { id } : {}) };
- const original = structuredClone(input);
- expect(Protocol.Event.parse(input)).toEqual(original);
- expect(input).toEqual(original);
- count += 1;
+ it.each([
+ ...NodeFs.readdirSync(fixtures).filter((name) => name.endsWith(".sse")),
+ "../experimental/analysis/fixtures/notebook-events.sse",
+ ])("parses events in %s unchanged", (name) => {
+ const stream = NodeFs.readFileSync(new NodeUrl.URL(name, fixtures), "utf8");
+ let count = 0;
+ for (const frame of stream.split(/\r?\n\r?\n/)) {
+ const event = /^event: (.+)$/m.exec(frame)?.[1];
+ if (!event && !/^data:/m.test(frame)) {
+ continue;
}
- expect(count).toBeGreaterThan(0);
- },
- );
+ const data = frame
+ .split(/\r?\n/)
+ .filter((line) => line.startsWith("data:"))
+ .map((line) => line.slice(5).trimStart())
+ .join("\n");
+ const id = /^id: (.+)$/m.exec(frame)?.[1];
+ const payload: unknown = JSON.parse(data);
+ const input = { event, data: payload, ...(id ? { id } : {}) };
+ const original = structuredClone(input);
+ expect(Protocol.Event.parse(input)).toEqual(original);
+ expect(input).toEqual(original);
+ count += 1;
+ }
+ expect(count).toBeGreaterThan(0);
+ });
});
describe("host protocol bindings", () => {
diff --git a/packages/client/src/operations.ts b/packages/client/src/operations.ts
index f83398ce7a..2ed64d985f 100644
--- a/packages/client/src/operations.ts
+++ b/packages/client/src/operations.ts
@@ -1,6 +1,7 @@
/* Copyright 2026 Marimo. All rights reserved. */
// Generated from protocols/host/openapi.yaml. Do not edit.
-// Regenerate with: pnpm --dir packages/client codegen
+// Experimental features: analysis
+// Regenerate with: pnpm --dir packages/client codegen --include-experimental=analysis
import * as Protocol from "./protocol.ts";
diff --git a/packages/client/src/protocol.ts b/packages/client/src/protocol.ts
index 401297e339..2ae9b7e3b6 100644
--- a/packages/client/src/protocol.ts
+++ b/packages/client/src/protocol.ts
@@ -1,6 +1,7 @@
/* Copyright 2026 Marimo. All rights reserved. */
// Generated from protocols/host/openapi.yaml. Do not edit.
-// Regenerate with: pnpm --dir packages/client codegen
+// Experimental features: analysis
+// Regenerate with: pnpm --dir packages/client codegen --include-experimental=analysis
import * as zod from "zod";
@@ -130,7 +131,6 @@ export const ObjectType = zod
"project",
"notebook",
"document",
- "analysis",
"cell",
"runtime",
"attachment",
@@ -140,9 +140,10 @@ export const ObjectType = zod
"display",
"variable",
"custom",
+ "analysis",
])
.describe(
- "The kinds a stream carries, and the values `types` accepts. The\ncatalog stream carries host, project, notebook, runtime, and\nattachment. A notebook's stream carries notebook, document,\nanalysis, cell, runtime, attachment, run, execution, console,\ndisplay, variable, and custom. Custom events describe occurrences\nrather than object state.\n",
+ "The kinds a stream carries, and the values `types` accepts. The\ncatalog stream carries host, project, notebook, runtime, and\nattachment. A notebook's stream carries notebook, document,\ncell, runtime, attachment, run, execution, console,\ndisplay, variable, and custom. Custom events describe occurrences\nrather than object state.\n",
);
export type ObjectType = zod.input<typeof ObjectType>;
@@ -503,7 +504,7 @@ export const Error = zod
kind: zod
.string()
.describe(
- "The producer's identifier, scoped by source. Open; this protocol defines `exception`, `interrupted`, and `internal` for executions, and reserves `multiple-definitions` and `cycle-dependencies` for analysis, because every host schedules them the same way. Clients show unrecognized kinds as plain errors.",
+ "The producer's identifier, scoped by source. Open; this protocol defines `exception`, `interrupted`, and `internal` for executions. Clients show unrecognized kinds as plain errors.",
),
message: zod
.string()
@@ -731,42 +732,6 @@ export const CellConfig = zod
export type CellConfig = zod.input<typeof CellConfig>;
export type CellConfigOutput = zod.output<typeof CellConfig>;
-export const CellAnalysis = zod
- .object({
- defs: zod.array(zod.string()).describe("Names this cell defines."),
- refs: zod
- .array(zod.string())
- .describe("Names this cell reads from other cells."),
- errors: zod
- .array(Error)
- .describe(
- "Diagnostics about the code. Only an `error` affects scheduling. `multiple-definitions` lists the definers in `related`, each `range` at its definition; `cycle-dependencies` lists the cycle. Both are reported on every cell involved, differing only in `range`. Every other kind belongs to the analyzer named in `source`.",
- ),
- })
- .describe(
- "What a cell's text implies: the names it defines and reads, which\nare the edges of the dependency graph, and the errors found in the\ntext alone or across the set of cells. A pure function of the\ncode, so two producers compute the same values.\n",
- );
-
-export type CellAnalysis = zod.input<typeof CellAnalysis>;
-export type CellAnalysisOutput = zod.output<typeof CellAnalysis>;
-
-export const Analysis = zod
- .object({
- notebook_id: NotebookId.describe("The notebook this analysis belongs to."),
- document_etag: Etag.describe(
- "The document revision this analysis describes. Equal to the document's `etag` while the analysis is current.",
- ),
- cells: zod
- .record(zod.string(), CellAnalysis)
- .describe("Every cell of that revision, by id."),
- })
- .describe(
- "What a document's cells imply, computed together over one\nrevision of the document so that cross-cell errors agree. One\nper notebook, and removed with it. It follows the document it\ndescribes, and `document_etag` says which revision that is.\n",
- );
-
-export type Analysis = zod.input<typeof Analysis>;
-export type AnalysisOutput = zod.output<typeof Analysis>;
-
export const Cell = zod
.object({
id: CellId,
@@ -990,42 +955,6 @@ export const DocumentEvent = zod
export type DocumentEvent = zod.input<typeof DocumentEvent>;
export type DocumentEventOutput = zod.output<typeof DocumentEvent>;
-export const AnalysisMessage = zod
- .object({
- request_id: zod
- .string()
- .optional()
- .describe(
- "The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress.",
- ),
- time: zod.iso
- .datetime({ offset: true })
- .optional()
- .describe(
- "When the host made this change. Present on live and replayed events; omitted on snapshots.",
- ),
- analysis: Analysis,
- })
- .describe("The data of an `analysis` event. The analysis as it now is.");
-
-export type AnalysisMessage = zod.input<typeof AnalysisMessage>;
-export type AnalysisMessageOutput = zod.output<typeof AnalysisMessage>;
-
-export const AnalysisEvent = zod
- .object({
- event: zod.enum(["analysis"]),
- id: zod
- .string()
- .min(1)
- .optional()
- .describe("The resume cursor. Omitted on snapshot and reset events."),
- data: AnalysisMessage,
- })
- .describe("A `analysis` event from a host stream.");
-
-export type AnalysisEvent = zod.input<typeof AnalysisEvent>;
-export type AnalysisEventOutput = zod.output<typeof AnalysisEvent>;
-
export const CellMessage = zod
.object({
request_id: zod
@@ -1559,13 +1488,86 @@ export const ReadyEvent = zod
export type ReadyEvent = zod.input<typeof ReadyEvent>;
export type ReadyEventOutput = zod.output<typeof ReadyEvent>;
+export const CellAnalysis = zod
+ .object({
+ defs: zod.array(zod.string()).describe("Names this cell defines."),
+ refs: zod
+ .array(zod.string())
+ .describe("Names this cell reads from other cells."),
+ errors: zod
+ .array(Error)
+ .describe(
+ "Diagnostics about the code. Only an `error` affects scheduling. `multiple-definitions` lists the definers in `related`, each `range` at its definition; `cycle-dependencies` lists the cycle. Both are reported on every cell involved, differing only in `range`. Every other kind belongs to the analyzer named in `source`.",
+ ),
+ })
+ .describe(
+ "Experimental (analysis). What a cell's text implies: the names it defines and reads, which\nare the edges of the dependency graph, and the errors found in the\ntext alone or across the set of cells. A pure function of the\ncode, so two producers compute the same values.\n",
+ );
+
+export type CellAnalysis = zod.input<typeof CellAnalysis>;
+export type CellAnalysisOutput = zod.output<typeof CellAnalysis>;
+
+export const Analysis = zod
+ .object({
+ notebook_id: NotebookId.describe("The notebook this analysis belongs to."),
+ document_etag: Etag.describe(
+ "The document revision this analysis describes. Equal to the document's `etag` while the analysis is current.",
+ ),
+ cells: zod
+ .record(zod.string(), CellAnalysis)
+ .describe("Every cell of that revision, by id."),
+ })
+ .describe(
+ "Experimental (analysis). What a document's cells imply, computed together over one\nrevision of the document so that cross-cell errors agree. One\nper notebook, and removed with it. It follows the document it\ndescribes, and `document_etag` says which revision that is.\n",
+ );
+
+export type Analysis = zod.input<typeof Analysis>;
+export type AnalysisOutput = zod.output<typeof Analysis>;
+
+export const AnalysisMessage = zod
+ .object({
+ request_id: zod
+ .string()
+ .optional()
+ .describe(
+ "The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress.",
+ ),
+ time: zod.iso
+ .datetime({ offset: true })
+ .optional()
+ .describe(
+ "When the host made this change. Present on live and replayed events; omitted on snapshots.",
+ ),
+ analysis: Analysis,
+ })
+ .describe(
+ "Experimental (analysis). The data of an `analysis` event. The analysis as it now is.",
+ );
+
+export type AnalysisMessage = zod.input<typeof AnalysisMessage>;
+export type AnalysisMessageOutput = zod.output<typeof AnalysisMessage>;
+
+export const AnalysisEvent = zod
+ .object({
+ event: zod.enum(["analysis"]),
+ id: zod
+ .string()
+ .min(1)
+ .optional()
+ .describe("The resume cursor. Omitted on snapshot and reset events."),
+ data: AnalysisMessage,
+ })
+ .describe("Experimental (analysis). A `analysis` event from a host stream.");
+
+export type AnalysisEvent = zod.input<typeof AnalysisEvent>;
+export type AnalysisEventOutput = zod.output<typeof AnalysisEvent>;
+
export const Event = zod
.union([
HostEvent,
ProjectEvent,
NotebookEvent,
DocumentEvent,
- AnalysisEvent,
CellEvent,
RuntimeEvent,
AttachmentEvent,
@@ -1587,6 +1589,7 @@ export const Event = zod
VariableRemovedEvent,
ResetEvent,
ReadyEvent,
+ AnalysisEvent,
])
.describe(
"A record, removal, or control message from a host stream.\nThe data is decoded JSON. Snapshot records and reset events omit\nthe cursor; ready and live events carry it. Unknown event names\nbelong to later versions or host extensions.\n",
@@ -1784,7 +1787,7 @@ export const CellInput = zod
config: CellConfigInput.optional(),
})
.describe(
- "A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags and the analysis.",
+ "A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags.",
);
export type CellInput = zod.input<typeof CellInput>;
diff --git a/protocols/host/README.md b/protocols/host/README.md
index 96a228b69d..83d7c44591 100644
--- a/protocols/host/README.md
+++ b/protocols/host/README.md
@@ -7,8 +7,13 @@ notebooks and runtimes.
- [OpenAPI schema](openapi.yaml)
- [JSON and SSE examples](fixtures/)
- [Conformance tests](conformance/README.md)
+- [Experimental features](experimental/README.md)
-The OpenAPI schema is the source of truth for the Python types in
+The base OpenAPI schema contains stable contracts. Experimental features live
+in `experimental/<feature>/`, with an additive OpenAPI Overlay, specification,
+fixtures, and conformance tests.
+
+The composed schema is the source of truth for the Python types in
`marimo/_core/protocol.py`, the typed event adapters in
`marimo/_core/protocol_events.py`, and the TypeScript schemas and types in
`packages/client/src/protocol.ts`. The frontend defines the native
@@ -26,6 +31,7 @@ Generate and check the Python bindings from the repository root:
```sh
make host-protocol-codegen
make host-protocol-check
+uv run --group test pytest tests/_lint/test_host_protocol_overlays.py
```
Generate and check the TypeScript bindings:
@@ -38,3 +44,29 @@ pnpm --dir packages/client test
Other consumers can generate types from the same schema by pinning a marimo
commit. Use the examples and conformance tests from that commit too.
+
+## Selecting experimental contracts
+
+marimo's checked-in Python and TypeScript bindings include all experimental
+features. The generation commands above preserve that selection.
+
+To generate stable-only bindings in a separate directory:
+
+```sh
+# Python types and event adapters
+uv run --no-config --locked --script protocols/host/generate.py --output-dir /tmp/host-python
+
+# TypeScript types, validation schemas, and operation definitions
+pnpm --dir packages/client codegen:stable --output-dir /tmp/host-typescript
+```
+
+Both generator scripts default to stable-only bindings. Add
+`--include-experimental` for all features or
+`--include-experimental=analysis` for a specific feature.
+Add `--check` to verify existing output without writing files.
+
+Stable-only generation covers the generated bindings; the client package
+itself currently requires experimental Analysis types.
+
+See [experimental features](experimental/README.md) for compatibility rules
+and instructions for adding features.
diff --git a/protocols/host/conformance/README.md b/protocols/host/conformance/README.md
index 5e1632d791..6107610d66 100644
--- a/protocols/host/conformance/README.md
+++ b/protocols/host/conformance/README.md
@@ -31,3 +31,15 @@ It does not yet cover opening an editor, answering host decisions, or unsupporte
operation responses. Retention expiry, external store changes, shutdown, and
transport failures also need implementation tests. WebSocket transport is outside
this contract.
+
+Experimental features have separate conformance tests. Check the stable
+contract and all experimental fixtures together with:
+
+```sh
+uv run --locked --project protocols/host/conformance \
+ pytest -c protocols/host/conformance/pyproject.toml \
+ protocols/host/conformance protocols/host/experimental
+```
+
+The base probes continue to ignore experimental events. Each feature's suite
+validates its payloads against the composed contract.
diff --git a/protocols/host/conformance/contract.py b/protocols/host/conformance/contract.py
index bb2d9b755b..8d0ccf6153 100644
--- a/protocols/host/conformance/contract.py
+++ b/protocols/host/conformance/contract.py
@@ -27,7 +27,6 @@ KINDS = (
"project",
"notebook",
"document",
- "analysis",
"cell",
"runtime",
"attachment",
diff --git a/protocols/host/conformance/pyproject.toml b/protocols/host/conformance/pyproject.toml
index 6766943813..32e9edc293 100644
--- a/protocols/host/conformance/pyproject.toml
+++ b/protocols/host/conformance/pyproject.toml
@@ -6,6 +6,7 @@ requires-python = ">=3.11"
dependencies = [
"httpx==0.28.1",
"jsonschema==4.26.0",
+ "python-jsonpath==2.2.1",
"msgspec[yaml]==0.22.0",
"psutil==7.2.2",
"pytest==9.0.3",
@@ -17,6 +18,7 @@ package = false
[tool.pytest.ini_options]
testpaths = ["."]
+pythonpath = [".", ".."]
addopts = "-ra"
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
diff --git a/protocols/host/conformance/test_contract.py b/protocols/host/conformance/test_contract.py
index 27eb82b78f..c87212839b 100644
--- a/protocols/host/conformance/test_contract.py
+++ b/protocols/host/conformance/test_contract.py
@@ -26,8 +26,6 @@ MODELS = {
"record.json": "HostRecord",
"cell.json": "Cell",
"cell-withheld.json": "Cell",
- "analysis.json": "Analysis",
- "diagnostic.json": "Error",
"edit.json": "EditRequest",
"edit-conflict-request.json": "EditRequest",
"edit-response.json": "EditResponse",
@@ -67,18 +65,6 @@ async def test_stream_examples(path):
frame.message()
[email protected]("message", [{}, {"analysis": {}}])
-async def test_invalid_analysis_messages_are_rejected(message):
- data = json.dumps(message)
- with pytest.raises(ValidationError):
- Frame(event="analysis", data=data).message()
-
- stream = f"event: analysis\ndata: {data}\n\nevent: ready\ndata: {{}}\n\n"
- watch = Watch(frames(lines(stream)), timeout=1)
- with pytest.raises(ValidationError):
- await watch.ready()
-
-
async def test_sse_preserves_multiline_data_comments_and_explicit_ids():
stream = ": ping\r\nid: 7\r\nevent: future\r\ndata: one\r\ndata: two\r\n\r\nevent: ready\ndata: {}\n\n"
assert [frame async for frame in frames(lines(stream))] == [
diff --git a/protocols/host/conformance/uv.lock b/protocols/host/conformance/uv.lock
index 1ab4a85217..aa4f8f7df0 100644
--- a/protocols/host/conformance/uv.lock
+++ b/protocols/host/conformance/uv.lock
@@ -135,6 +135,7 @@ dependencies = [
{ name = "psutil" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
+ { name = "python-jsonpath" },
]
[package.metadata]
@@ -145,6 +146,7 @@ requires-dist = [
{ name = "psutil", specifier = "==7.2.2" },
{ name = "pytest", specifier = "==9.0.3" },
{ name = "pytest-asyncio", specifier = "==1.3.0" },
+ { name = "python-jsonpath", specifier = "==2.2.1" },
]
[[package]]
@@ -315,6 +317,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" },
]
+[[package]]
+name = "python-jsonpath"
+version = "2.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ee/65/5dcdbd783b98a936f6aacee699aed4c0978d74cb0f8ad61dc59bb39b809a/python_jsonpath-2.2.1.tar.gz", hash = "sha256:3de7862cbc56566b61c2c25fe5a398bbba9466e972c1069a4c82cb60a8034dc5", size = 51022, upload-time = "2026-07-07T09:56:08.062Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/c1/e8ccd211bc8c411eb84341ae3ded659e6bdb926697fb7724e2dbbf8077cf/python_jsonpath-2.2.1-py3-none-any.whl", hash = "sha256:ff3a65df52336fec13e2728612d6e2a76221264762a1b5385fac5dd21ae6f40e", size = 65235, upload-time = "2026-07-07T09:56:06.755Z" },
+]
+
[[package]]
name = "pyyaml"
version = "6.0.3"
diff --git a/protocols/host/experimental/README.md b/protocols/host/experimental/README.md
new file mode 100644
index 0000000000..4bd6b7968c
--- /dev/null
+++ b/protocols/host/experimental/README.md
@@ -0,0 +1,36 @@
+# Experimental host protocol features
+
+Experimental features provide a way to develop and validate proposed host APIs
+before committing to a stable contract.
+
+Each feature lives in `experimental/<feature>/` with an OpenAPI overlay,
+specification, fixtures, and conformance tests. Consumers explicitly
+[select the features](../README.md#selecting-experimental-contracts) they support.
+
+- Overlays **MUST NOT** change stable contracts. Review the composed
+ specification to verify this.
+- Stable clients **MUST** remain correct without opting in.
+- Tests **SHOULD** verify feature behavior and compatibility with stable clients.
+- Experimental contracts **MAY** change or be removed between releases.
+
+Run `make host-protocol-check` and `pnpm --dir packages/client test --run`
+to check generation, fixtures, and stable-client compatibility. Each feature
+must also generate a valid contract when selected on its own.
+
+Once implementation experience establishes a stable design, move its contract,
+documentation, fixtures, and tests into the core protocol.
+
+## Composing the schema
+
+Both language generators use `generate.py` to compose the contract. To inspect
+the OpenAPI schema with selected features as JSON:
+
+```sh
+uv run --no-config --locked --script protocols/host/generate.py --schema --include-experimental=analysis > /tmp/host-openapi.json
+```
+
+Separate multiple feature names with commas; unknown names fail.
+The composer applies Overlay 1.0 `update` actions in feature-name order.
+JSONPath targets must match an object or array: object updates merge
+recursively, and array updates append an entry. Root updates (`target: $`)
+can add paths and components together. Removal actions are not supported.
diff --git a/protocols/host/experimental/analysis/conformance/test_analysis.py b/protocols/host/experimental/analysis/conformance/test_analysis.py
new file mode 100644
index 0000000000..ba6c10ed8b
--- /dev/null
+++ b/protocols/host/experimental/analysis/conformance/test_analysis.py
@@ -0,0 +1,89 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Analysis's experimental contract and its compatibility with stable streams."""
+
+import json
+from collections.abc import AsyncIterator
+from pathlib import Path
+from typing import Any
+
+import pytest
+from contract import Frame, Watch, frames
+from generate import compose
+from jsonschema import Draft202012Validator, FormatChecker, ValidationError
+
+FIXTURES = Path(__file__).resolve().parent.parent / "fixtures"
+SPEC = compose("analysis")
+
+
+def validate(name: str, value: Any) -> None:
+ Draft202012Validator(
+ {
+ "$ref": f"#/components/schemas/{name}",
+ "components": SPEC["components"],
+ },
+ format_checker=FormatChecker(),
+ ).validate(value)
+
+
+async def lines(text: str) -> AsyncIterator[str]:
+ for line in text.splitlines():
+ yield line
+
+
+ ("filename", "schema"),
+ [("analysis.json", "Analysis"), ("diagnostic.json", "Error")],
+)
+def test_examples(filename: str, schema: str) -> None:
+ validate(schema, json.loads((FIXTURES / filename).read_text()))
+
+
+def test_schemas_are_valid() -> None:
+ for schema in SPEC["components"]["schemas"].values():
+ Draft202012Validator.check_schema(schema)
+
+
[email protected]("message", [{}, {"analysis": {}}])
+def test_invalid_analysis_messages_are_rejected(message: Any) -> None:
+ with pytest.raises(ValidationError):
+ validate("AnalysisMessage", message)
+
+
+async def test_analysis_follows_document_and_matches_revision() -> None:
+ document = None
+ found = []
+ async for frame in frames(
+ lines((FIXTURES / "notebook-events.sse").read_text())
+ ):
+ if frame.comment is not None:
+ continue
+ message = json.loads(frame.data)
+ validate(
+ "Event",
+ {
+ "event": frame.event,
+ "data": message,
+ **({"id": frame.id} if frame.id is not None else {}),
+ },
+ )
+ if frame.event == "document":
+ document = message["document"]
+ elif frame.event == "analysis":
+ analysis = message["analysis"]
+ assert document is not None
+ assert analysis["document_etag"] == document["etag"]
+ assert analysis["notebook_id"] == document["notebook_id"]
+ assert set(analysis["cells"]) == set(document["order"])
+ found.append(analysis)
+ assert found == [json.loads((FIXTURES / "analysis.json").read_text())]
+
+
+async def test_stable_probe_ignores_experimental_payloads() -> None:
+ # A stable consumer must not validate a contract it has not selected.
+ stream = (
+ "id: 1\nevent: analysis\ndata: {}\n\nid: 1\nevent: ready\ndata: {}\n\n"
+ )
+ assert await Watch(frames(lines(stream)), timeout=1).ready() == [
+ Frame(event="analysis", data="{}", id="1"),
+ Frame(event="ready", data="{}", id="1"),
+ ]
diff --git a/protocols/host/fixtures/analysis.json b/protocols/host/experimental/analysis/fixtures/analysis.json
similarity index 100%
rename from protocols/host/fixtures/analysis.json
rename to protocols/host/experimental/analysis/fixtures/analysis.json
diff --git a/protocols/host/fixtures/diagnostic.json b/protocols/host/experimental/analysis/fixtures/diagnostic.json
similarity index 100%
rename from protocols/host/fixtures/diagnostic.json
rename to protocols/host/experimental/analysis/fixtures/diagnostic.json
diff --git a/protocols/host/experimental/analysis/fixtures/notebook-events.sse b/protocols/host/experimental/analysis/fixtures/notebook-events.sse
new file mode 100644
index 0000000000..f0c36621f1
--- /dev/null
+++ b/protocols/host/experimental/analysis/fixtures/notebook-events.sse
@@ -0,0 +1,50 @@
+event: notebook
+data: {"notebook":{"id":"n1","project_id":"p1","path":"forecasts.py","name":"forecasts","operations":["notebook.update","notebook.delete","notebook.open","notebook.edit","notebook.save","runtime.start","runtime.stop","runtime.run","runtime.interrupt"],"updated_at":"2026-10-01T09:00:00Z"}}
+
+event: cell
+data: {"cell":{"id":"c1","notebook_id":"n1","code":"import marimo as mo","name":"_","config":{"column":null,"disabled":false,"hide_code":false,"expand_output":false},"etag":"48d6363150d6281c","code_hash":"f649a0b56c5c1966","version":1}}
+
+event: cell
+data: {"cell":{"id":"c3","notebook_id":"n1","code":"x = 21","name":"_","config":{"column":null,"disabled":false,"hide_code":false,"expand_output":false},"etag":"5cf78587dcf22d8d","code_hash":"0f71456db2ea83f4","version":1}}
+
+event: cell
+data: {"cell":{"id":"k7f2","notebook_id":"n1","code":"y = x * 2","name":"_","config":{"column":null,"disabled":false,"hide_code":true,"expand_output":false},"etag":"f199b5f48ae0aabb","code_hash":"db2b9ee97e1ecf4c","version":1}}
+
+event: document
+data: {"document":{"notebook_id":"n1","order":["c1","c3","k7f2"],"app_config":{"width":"full"},"etag":"e59d9c2af02f5814","saved_etag":"e59d9c2af02f5814","order_etag":"b9c412c2d7d63161","app_config_etag":"283513048dfeb45b","version":1}}
+
+event: analysis
+data: {"analysis":{"notebook_id":"n1","document_etag":"e59d9c2af02f5814","cells":{"c1":{"defs":["mo"],"refs":[],"errors":[]},"c3":{"defs":["x"],"refs":[],"errors":[]},"k7f2":{"defs":["y"],"refs":["x"],"errors":[]}}}}
+
+event: runtime
+data: {"runtime":{"id":"r1","notebook_id":"n1","status":"running","message":null,"marimo_version":"0.25.0","started_at":"2026-10-01T09:14:02Z"}}
+
+event: attachment
+data: {"attachment":{"id":"a1","notebook_id":"n1","kind":"browser","name":"Chrome","since":"2026-10-01T09:14:30Z"}}
+
+event: run
+data: {"run":{"id":"run-scratch","notebook_id":"n1","status":"running","started_at":"2026-10-01T09:20:01Z","completed_at":null,"caused_by":null,"outcome":null},"request_id":"7c1e9a3f-5b2d-4e8a-9f10-3c6d7e8f9a0b"}
+
+event: execution
+data: {"execution":{"id":"e7","run_id":"run-scratch","cell_id":null,"code_hash":"cebd60abaf00acdc","depends_on":{},"status":"running","outcome":{"kind":"value","display":{"mimetype":"text/html","data":"<span class=\"markdown\">working…</span>"}},"started_at":"2026-10-01T09:20:01Z","completed_at":null,"runtime_id":"r1","caused_by":null,"code":"import marimo as mo\nmo.output.append(mo.md(\"working…\"))\nprint(1 + 1)\n3 * 3"},"request_id":"7c1e9a3f-5b2d-4e8a-9f10-3c6d7e8f9a0b"}
+
+event: console
+data: {"console":{"id":"o3","execution_id":"e7","seq":0,"name":"stdout","text":"2\n"},"request_id":"7c1e9a3f-5b2d-4e8a-9f10-3c6d7e8f9a0b"}
+
+event: attachment
+data: {"attachment":{"id":"a2","notebook_id":"n1","kind":"agent","name":"Notebook CLI","since":"2026-10-01T09:20:03Z"}}
+
+id: 95
+event: ready
+data: {}
+
+: ping
+
+id: 96
+event: execution
+data: {"execution":{"id":"e7","run_id":"run-scratch","cell_id":null,"code_hash":"cebd60abaf00acdc","depends_on":{},"status":"completed","outcome":{"kind":"value","display":{"mimetype":"text/html","data":"<span class=\"markdown\">working…</span><pre>9</pre>"}},"started_at":"2026-10-01T09:20:01Z","completed_at":"2026-10-01T09:20:04Z","runtime_id":"r1","caused_by":null,"code":"import marimo as mo\nmo.output.append(mo.md(\"working…\"))\nprint(1 + 1)\n3 * 3"},"request_id":"7c1e9a3f-5b2d-4e8a-9f10-3c6d7e8f9a0b","time":"2026-10-01T09:20:04Z"}
+
+id: 97
+event: run
+data: {"run":{"id":"run-scratch","notebook_id":"n1","status":"completed","started_at":"2026-10-01T09:20:01Z","completed_at":"2026-10-01T09:20:04Z","caused_by":null,"outcome":"succeeded"},"request_id":"7c1e9a3f-5b2d-4e8a-9f10-3c6d7e8f9a0b","time":"2026-10-01T09:20:04Z"}
+
diff --git a/protocols/host/experimental/analysis/overlay.yaml b/protocols/host/experimental/analysis/overlay.yaml
new file mode 100644
index 0000000000..a582ce9e6a
--- /dev/null
+++ b/protocols/host/experimental/analysis/overlay.yaml
@@ -0,0 +1,80 @@
+overlay: 1.0.0
+info:
+ title: Experimental document analysis
+ version: 0.1.0
+extends: ../../openapi.yaml
+actions:
+ - target: $.components.schemas
+ update:
+ Analysis:
+ type: object
+ description: |
+ What a document's cells imply, computed together over one
+ revision of the document so that cross-cell errors agree. One
+ per notebook, and removed with it. It follows the document it
+ describes, and `document_etag` says which revision that is.
+ required: [notebook_id, document_etag, cells]
+ properties:
+ notebook_id:
+ $ref: '#/components/schemas/NotebookId'
+ description: The notebook this analysis belongs to.
+ document_etag:
+ $ref: '#/components/schemas/Etag'
+ description: The document revision this analysis describes. Equal to the document's `etag` while the analysis is current.
+ cells:
+ type: object
+ additionalProperties: { $ref: '#/components/schemas/CellAnalysis' }
+ description: Every cell of that revision, by id.
+ CellAnalysis:
+ type: object
+ description: |
+ What a cell's text implies: the names it defines and reads, which
+ are the edges of the dependency graph, and the errors found in the
+ text alone or across the set of cells. A pure function of the
+ code, so two producers compute the same values.
+ required: [defs, refs, errors]
+ properties:
+ defs:
+ type: array
+ items: { type: string }
+ description: Names this cell defines.
+ refs:
+ type: array
+ items: { type: string }
+ description: Names this cell reads from other cells.
+ errors:
+ type: array
+ items: { $ref: '#/components/schemas/Error' }
+ description: Diagnostics about the code. Only an `error` affects scheduling. `multiple-definitions` lists the definers in `related`, each `range` at its definition; `cycle-dependencies` lists the cycle. Both are reported on every cell involved, differing only in `range`. Every other kind belongs to the analyzer named in `source`.
+ AnalysisEvent:
+ type: object
+ description: A `analysis` event from a host stream.
+ required: [event, data]
+ properties:
+ event: { type: string, enum: [analysis] }
+ id:
+ type: string
+ minLength: 1
+ description: The resume cursor. Omitted on snapshot and reset events.
+ data: { $ref: '#/components/schemas/AnalysisMessage' }
+ AnalysisMessage:
+ type: object
+ description: The data of an `analysis` event. The analysis as it now is.
+ required: [analysis]
+ properties:
+ request_id:
+ type: string
+ description: The `Idempotency-Key` of the request that caused this event. Omitted when no request did, and on snapshots unless that request's work is still in progress.
+ time:
+ type: string
+ format: date-time
+ description: When the host made this change. Present on live and replayed events; omitted on snapshots.
+ analysis: { $ref: '#/components/schemas/Analysis' }
+ - target: $.components.schemas.ObjectType.enum
+ update: analysis
+ - target: $.components.schemas.Event.oneOf
+ update:
+ $ref: '#/components/schemas/AnalysisEvent'
+ - target: $.components.schemas.Event.discriminator.mapping
+ update:
+ analysis: '#/components/schemas/AnalysisEvent'
diff --git a/protocols/host/experimental/analysis/spec.md b/protocols/host/experimental/analysis/spec.md
new file mode 100644
index 0000000000..3f4580c239
--- /dev/null
+++ b/protocols/host/experimental/analysis/spec.md
@@ -0,0 +1,21 @@
+# Document analysis
+
+An **analysis** describes a document's definitions, references, and problems,
+so clients can understand its dependency graph and identify problems before
+execution. Because editing one cell can affect others, analysis covers the
+whole document at one revision.
+
+An analysis is current while its `document_etag` matches the document's `etag`.
+A change to that etag makes the analysis stale, including results for cells
+whose source did not change. Execution and runtime restarts do not themselves
+invalidate analysis. Clients MUST NOT treat stale analysis as describing the
+current document.
+
+Hosts MAY publish analysis on the notebook stream, after the document it
+describes. Each analysis replaces the previous one in full and is retained
+until replaced or the notebook is removed. Snapshots include the latest
+available analysis, which may be stale. Computation may finish after `ready`;
+receiving `ready` does not guarantee analysis of the current document.
+
+The [overlay](overlay.yaml) defines the message formats and diagnostics.
+The [examples](fixtures/) show complete messages.
diff --git a/protocols/host/fixtures/README.md b/protocols/host/fixtures/README.md
index de855ba5d7..0b869214b9 100644
--- a/protocols/host/fixtures/README.md
+++ b/protocols/host/fixtures/README.md
@@ -9,7 +9,7 @@ opaque. Gaps represent events outside an excerpt or its filter.
shown in `run.sse`. Its snapshot includes the display and console output
already produced. Both streams end with the same execution and run completion
events, using the same cursors. `cell.json` is cell `c3` from that snapshot,
-and `analysis.json` is the analysis of that document. `cell-withheld.json`
+and `cell-withheld.json`
is the same cell with source withheld from the client: its hashes and version
are unchanged.
@@ -48,6 +48,6 @@ from storage. Its source is `x = 21`, but its original run and runtime are
unknown. It illustrates a separate restoration scenario, outside the session
above.
-`diagnostic.json` illustrates a separate document in which cells `c1` and `c2`
-both define `x`. The other problem responses illustrate individual rejected
-requests. `record.json` contains fictional local discovery credentials.
+The problem responses illustrate individual rejected requests. `record.json` contains fictional local discovery credentials.
+
+Experimental Analysis examples live in [experimental/analysis](../experimental/analysis/spec.md).
diff --git a/protocols/host/fixtures/notebook-events.sse b/protocols/host/fixtures/notebook-events.sse
index f0c36621f1..e5e42bbbd4 100644
--- a/protocols/host/fixtures/notebook-events.sse
+++ b/protocols/host/fixtures/notebook-events.sse
@@ -13,9 +13,6 @@ data: {"cell":{"id":"k7f2","notebook_id":"n1","code":"y = x * 2","name":"_","con
event: document
data: {"document":{"notebook_id":"n1","order":["c1","c3","k7f2"],"app_config":{"width":"full"},"etag":"e59d9c2af02f5814","saved_etag":"e59d9c2af02f5814","order_etag":"b9c412c2d7d63161","app_config_etag":"283513048dfeb45b","version":1}}
-event: analysis
-data: {"analysis":{"notebook_id":"n1","document_etag":"e59d9c2af02f5814","cells":{"c1":{"defs":["mo"],"refs":[],"errors":[]},"c3":{"defs":["x"],"refs":[],"errors":[]},"k7f2":{"defs":["y"],"refs":["x"],"errors":[]}}}}
-
event: runtime
data: {"runtime":{"id":"r1","notebook_id":"n1","status":"running","message":null,"marimo_version":"0.25.0","started_at":"2026-10-01T09:14:02Z"}}
diff --git a/protocols/host/generate.py b/protocols/host/generate.py
index a1619613c6..1324c768f7 100644
--- a/protocols/host/generate.py
+++ b/protocols/host/generate.py
@@ -4,19 +4,90 @@
# "datamodel-code-generator==0.72.0",
# "jinja2==3.1.6",
# "ruff==0.16.4",
-# "msgspec[yaml]==0.22.0"
+# "msgspec[yaml]==0.22.0",
+# "python-jsonpath==2.2.1"
# ]
# ///
-"""Generate msgspec models for the marimo host protocol."""
+"""Compose the host contract and generate its Python bindings or OpenAPI JSON."""
import argparse
import ast
+import copy
+import json
import subprocess
+import sys
import tempfile
from collections import defaultdict
from pathlib import Path
from typing import Any
+import jsonpath
+import msgspec.yaml
+
+ROOT = Path(__file__).resolve().parent
+
+
+def merge(target: dict[str, Any], update: dict[str, Any]) -> None:
+ """Recursively merge object properties, replacing other values."""
+ for key, value in update.items():
+ if isinstance(target.get(key), dict) and isinstance(value, dict):
+ merge(target[key], value)
+ else:
+ target[key] = copy.deepcopy(value)
+
+
+def compose(
+ selection: str | None = None, *, root: Path = ROOT
+) -> dict[str, Any]:
+ document = msgspec.yaml.decode((root / "openapi.yaml").read_bytes())
+ available = {
+ path.parent.name: path
+ for path in (root / "experimental").glob("*/overlay.yaml")
+ }
+ if selection is None:
+ selected = []
+ elif selection == "all":
+ selected = sorted(available)
+ else:
+ selected = sorted(set(selection.split(",")))
+ unknown = set(selected) - available.keys()
+ if unknown:
+ raise ValueError(
+ f"Unknown experimental features: {', '.join(sorted(unknown))}"
+ )
+ for name in selected:
+ overlay = msgspec.yaml.decode(available[name].read_bytes())
+ if overlay["overlay"] != "1.0.0":
+ raise ValueError("Only OpenAPI Overlay 1.0.0 is supported")
+ schemas = document["components"]["schemas"]
+ previous = set(schemas)
+ for action in overlay["actions"]:
+ if action.get("remove"):
+ raise ValueError("Experimental overlays support updates only")
+ matches = jsonpath.findall(action["target"], document, strict=True)
+ if not matches:
+ raise ValueError(
+ f"Overlay target does not exist: {action['target']}"
+ )
+ for target in matches:
+ update = action["update"]
+ if isinstance(target, dict) and isinstance(update, dict):
+ merge(target, update)
+ elif isinstance(target, list):
+ target.append(copy.deepcopy(update))
+ else:
+ raise ValueError(
+ f"Expected an object or array: {action['target']}"
+ )
+ for schema in schemas.keys() - previous:
+ schemas[schema]["description"] = (
+ f"Experimental ({name}). "
+ + schemas[schema].get("description", "")
+ )
+ if selected:
+ document["x-marimo-experimental"] = selected
+ return document
+
# msgspec decodes tuple fields from JSON arrays without changing the wire format.
class TupleArrays(ast.NodeTransformer):
@@ -77,7 +148,17 @@ def tuple_arrays(source: bytes) -> bytes:
return source
-def event_adapters(document: dict[str, Any]) -> str:
+def regeneration_command(document: dict[str, Any]) -> str:
+ features = document.get("x-marimo-experimental", [])
+ command = "uv run --no-config --locked --script protocols/host/generate.py"
+ return command + (
+ f" --include-experimental={','.join(features)}" if features else ""
+ )
+
+
+def event_adapters(
+ document: dict[str, Any], *, relative_import: bool = False
+) -> str:
"""Derive typed event adapters from the discriminator and schema refs."""
from jinja2 import Environment, FileSystemLoader, StrictUndefined
@@ -135,7 +216,11 @@ def event_adapters(document: dict[str, Any]) -> str:
keep_trailing_newline=True,
)
return env.get_template("events.py.jinja2").render(
- records=list(records.values()), removals=removed
+ records=list(records.values()),
+ removals=removed,
+ command=regeneration_command(document),
+ features=document.get("x-marimo-experimental", []),
+ relative_import=relative_import,
)
@@ -163,6 +248,10 @@ def generate_models(document: dict[str, Any], output_path: Path) -> None:
)
# Freeze records and keep identifier types distinct without changing their wire shape.
+ features = document.get("x-marimo-experimental", [])
+ experimental_header = (
+ f"# Experimental features: {', '.join(features)}\n" if features else ""
+ )
schemas = document["components"]["schemas"]
template_data = defaultdict(dict)
for name, schema in schemas.items():
@@ -193,7 +282,8 @@ def generate_models(document: dict[str, Any], output_path: Path) -> None:
"# Copyright 2026 Marimo. All rights reserved.\n"
"# AUTO-GENERATED FILE — DO NOT EDIT.\n"
"# Generated from protocols/host/openapi.yaml.\n"
- "# Regenerate with `make host-protocol-codegen`.\n"
+ + experimental_header
+ + f"# Regenerate with `{regeneration_command(document)}`.\n"
"# ruff: noqa: TC003 - msgspec resolves annotations at runtime\n"
"# ruff: noqa: RUF036 - the generator orders unions; the wire shape is the same"
),
@@ -205,21 +295,42 @@ def generate_models(document: dict[str, Any], output_path: Path) -> None:
def main() -> None:
- import msgspec.yaml
-
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--check", action="store_true")
- args = parser.parse_args()
- root = Path(__file__).resolve().parent
- destination = root.parents[1] / "marimo" / "_core"
- document = sorted_keys(
- msgspec.yaml.decode((root / "openapi.yaml").read_bytes())
+ mode = parser.add_mutually_exclusive_group()
+ mode.add_argument("--check", action="store_true")
+ mode.add_argument(
+ "--schema",
+ action="store_true",
+ help="Print the composed OpenAPI document as JSON instead of generating bindings",
+ )
+ parser.add_argument(
+ "--include-experimental",
+ nargs="?",
+ const="all",
+ help="Include all experimental features, or a comma-separated selection",
)
+ parser.add_argument("--output-dir", type=Path)
+ args = parser.parse_args()
+ default_destination = ROOT.parents[1] / "marimo" / "_core"
+ destination = args.output_dir or default_destination
+ document = compose(args.include_experimental)
+ if args.schema:
+ sys.stdout.write(json.dumps(document) + "\n")
+ return
+ document = sorted_keys(document)
+ if not args.check:
+ destination.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as directory:
output = Path(directory) / "protocol.py"
generate_models(document, output)
adapters = Path(directory) / "protocol_events.py"
- adapters.write_text(event_adapters(document), encoding="utf-8")
+ adapters.write_text(
+ event_adapters(
+ document,
+ relative_import=destination.resolve() != default_destination,
+ ),
+ encoding="utf-8",
+ )
format_generated(adapters)
for generated in (output, adapters):
target = destination / generated.name
@@ -227,7 +338,8 @@ def main() -> None:
if not target.exists() or target.read_bytes() != content:
if args.check:
raise SystemExit(
- "Host bindings are stale; run make host-protocol-codegen"
+ "Host bindings are stale; regenerate with the same "
+ "feature selection and output directory"
)
target.write_bytes(content)
print("Host protocol models are current.")
diff --git a/protocols/host/generate.py.lock b/protocols/host/generate.py.lock
index 88efab3624..0201ed852b 100644
--- a/protocols/host/generate.py.lock
+++ b/protocols/host/generate.py.lock
@@ -6,19 +6,12 @@ resolution-markers = [
"python_full_version < '3.14'",
]
-[options]
-exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
-exclude-newer-span = "P7D"
-
-[options.exclude-newer-package]
-agent-plugins = false
-marimo-lens = false
-
[manifest]
requirements = [
{ name = "datamodel-code-generator", specifier = "==0.72.0" },
{ name = "jinja2", specifier = "==3.1.6" },
{ name = "msgspec", extras = ["yaml"], specifier = "==0.22.0" },
+ { name = "python-jsonpath", specifier = "==2.2.1" },
{ name = "ruff", specifier = "==0.16.4" },
]
@@ -522,6 +515,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/db/1d/068464f23075f66a8f1b806935e9cd9363ee446636ea70d2c22ee8659dbf/pydantic_core-2.46.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266", size = 2140686, upload-time = "2026-08-28T10:01:28.947Z" },
]
+[[package]]
+name = "python-jsonpath"
+version = "2.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ee/65/5dcdbd783b98a936f6aacee699aed4c0978d74cb0f8ad61dc59bb39b809a/python_jsonpath-2.2.1.tar.gz", hash = "sha256:3de7862cbc56566b61c2c25fe5a398bbba9466e972c1069a4c82cb60a8034dc5", size = 51022, upload-time = "2026-07-07T09:56:08.062Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/c1/e8ccd211bc8c411eb84341ae3ded659e6bdb926697fb7724e2dbbf8077cf/python_jsonpath-2.2.1-py3-none-any.whl", hash = "sha256:ff3a65df52336fec13e2728612d6e2a76221264762a1b5385fac5dd21ae6f40e", size = 65235, upload-time = "2026-07-07T09:56:06.755Z" },
+]
+
[[package]]
name = "pytokens"
version = "0.4.1"
diff --git a/protocols/host/openapi.yaml b/protocols/host/openapi.yaml
index ff60dafea1..1e9d2c75df 100644
--- a/protocols/host/openapi.yaml
+++ b/protocols/host/openapi.yaml
@@ -220,8 +220,7 @@ paths:
that orders them, in the opening phase and after every change, so
a client never holds a document whose `order` names a cell it has
not received. The effects of one change to the document arrive
- together, with no other event between them. The analysis, when
- the host has one, follows the document it describes.
+ together, with no other event between them.
parameters:
- $ref: '#/components/parameters/NotebookId'
- $ref: '#/components/parameters/Types'
@@ -239,7 +238,7 @@ paths:
description: |
One event. `data` holds JSON, and the event name says
which shape: a NotebookMessage, DocumentMessage,
- AnalysisMessage, CellMessage, RuntimeMessage, AttachmentMessage,
+ CellMessage, RuntimeMessage, AttachmentMessage,
RunMessage, ExecutionMessage, ConsoleMessage,
DisplayMessage, or VariableMessage for an event named
after that kind; a CustomMessage for `custom`; a RemovedMessage
@@ -294,10 +293,8 @@ paths:
On success the document has changed, and every attachment sees
the effects on the notebook's stream with this request's key: the
cells that changed, `cell.removed` for each deleted cell, and then
- the document, with nothing in between. The analysis of the new
- revision follows with the same key once the host has it. The
- response carries the new etags, as of this edit or a later
- revision, so a client keeps editing without reading again. A
+ the document, with nothing in between. The response carries the new
+ etags, as of this edit or a later revision, so a client keeps editing without reading again. A
change that changes nothing keeps its etags and sends no events.
Whether the result has reached the store shows in the document's
`saved_etag`.
@@ -832,10 +829,10 @@ components:
The kinds a stream carries, and the values `types` accepts. The
catalog stream carries host, project, notebook, runtime, and
attachment. A notebook's stream carries notebook, document,
- analysis, cell, runtime, attachment, run, execution, console,
+ cell, runtime, attachment, run, execution, console,
display, variable, and custom. Custom events describe occurrences
rather than object state.
- enum: [host, project, notebook, document, analysis, cell, runtime, attachment, run, execution, console, display, variable, custom]
+ enum: [host, project, notebook, document, cell, runtime, attachment, run, execution, console, display, variable, custom]
Custom:
type: object
description: |
@@ -1137,7 +1134,7 @@ components:
properties:
kind:
type: string
- description: The producer's identifier, scoped by source. Open; this protocol defines `exception`, `interrupted`, and `internal` for executions, and reserves `multiple-definitions` and `cycle-dependencies` for analysis, because every host schedules them the same way. Clients show unrecognized kinds as plain errors.
+ description: The producer's identifier, scoped by source. Open; this protocol defines `exception`, `interrupted`, and `internal` for executions. Clients show unrecognized kinds as plain errors.
message:
type: string
description: A description of the error to show people.
@@ -1318,46 +1315,6 @@ components:
expand_output:
type: boolean
description: Whether clients show the full output rather than a scrolled box.
- Analysis:
- type: object
- description: |
- What a document's cells imply, computed together over one
- revision of the document so that cross-cell errors agree. One
- per notebook, and removed with it. It follows the document it
- describes, and `document_etag` says which revision that is.
- required: [notebook_id, document_etag, cells]
- properties:
- notebook_id:
- $ref: '#/components/schemas/NotebookId'
- description: The notebook this analysis belongs to.
- document_etag:
- $ref: '#/components/schemas/Etag'
- description: The document revision this analysis describes. Equal to the document's `etag` while the analysis is current.
- cells:
- type: object
- additionalProperties: { $ref: '#/components/schemas/CellAnalysis' }
- description: Every cell of that revision, by id.
- CellAnalysis:
- type: object
- description: |
- What a cell's text implies: the names it defines and reads, which
- are the edges of the dependency graph, and the errors found in the
- text alone or across the set of cells. A pure function of the
- code, so two producers compute the same values.
- required: [defs, refs, errors]
- properties:
- defs:
- type: array
- items: { type: string }
- description: Names this cell defines.
- refs:
- type: array
- items: { type: string }
- description: Names this cell reads from other cells.
- errors:
- type: array
- items: { $ref: '#/components/schemas/Error' }
- description: Diagnostics about the code. Only an `error` affects scheduling. `multiple-definitions` lists the definers in `related`, each `range` at its definition; `cycle-dependencies` lists the cycle. Both are reported on every cell involved, differing only in `range`. Every other kind belongs to the analyzer named in `source`.
Cell:
type: object
description: |
@@ -1434,7 +1391,6 @@ components:
- $ref: '#/components/schemas/ProjectEvent'
- $ref: '#/components/schemas/NotebookEvent'
- $ref: '#/components/schemas/DocumentEvent'
- - $ref: '#/components/schemas/AnalysisEvent'
- $ref: '#/components/schemas/CellEvent'
- $ref: '#/components/schemas/RuntimeEvent'
- $ref: '#/components/schemas/AttachmentEvent'
@@ -1463,7 +1419,6 @@ components:
project: '#/components/schemas/ProjectEvent'
notebook: '#/components/schemas/NotebookEvent'
document: '#/components/schemas/DocumentEvent'
- analysis: '#/components/schemas/AnalysisEvent'
cell: '#/components/schemas/CellEvent'
runtime: '#/components/schemas/RuntimeEvent'
attachment: '#/components/schemas/AttachmentEvent'
@@ -1520,14 +1475,6 @@ components:
event: { type: string, enum: [document] }
id: *event_cursor
data: { $ref: '#/components/schemas/DocumentMessage' }
- AnalysisEvent:
- type: object
- description: A `analysis` event from a host stream.
- required: [event, data]
- properties:
- event: { type: string, enum: [analysis] }
- id: *event_cursor
- data: { $ref: '#/components/schemas/AnalysisMessage' }
CellEvent:
type: object
description: A `cell` event from a host stream.
@@ -1800,14 +1747,6 @@ components:
request_id: *request_id
time: *time
document: { $ref: '#/components/schemas/Document' }
- AnalysisMessage:
- type: object
- description: The data of an `analysis` event. The analysis as it now is.
- required: [analysis]
- properties:
- request_id: *request_id
- time: *time
- analysis: { $ref: '#/components/schemas/Analysis' }
CellMessage:
type: object
description: The data of a `cell` event. The cell as it now is.
@@ -1966,7 +1905,7 @@ components:
default: false
CellInput:
type: object
- description: A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags and the analysis.
+ description: A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags.
required: [id, code]
properties:
id:
diff --git a/protocols/host/spec.md b/protocols/host/spec.md
index f1c758673e..34c0d2fb49 100644
--- a/protocols/host/spec.md
+++ b/protocols/host/spec.md
@@ -96,13 +96,6 @@ presentation preference; it does not withhold source. A null `code` field does
not promise that source cannot be inferred from hashes, outputs, diagnostics,
or other protocol data.
-A host MAY also publish an **analysis** of the document: each cell's
-definitions, references, and problems, so clients can understand a cell's role
-in the computation and identify problems before execution. Because cells share
-a dependency graph, editing one can affect the analysis of others. An analysis
-therefore covers the whole document and names the revision it describes. Hosts
-publish it after that revision of the document.
-
### Runtimes and attachments
Clients connected to a notebook share its document and runtime. Hosts MUST
diff --git a/protocols/host/templates/events.py.jinja2 b/protocols/host/templates/events.py.jinja2
index 8e30625c0d..9029c1ac58 100644
--- a/protocols/host/templates/events.py.jinja2
+++ b/protocols/host/templates/events.py.jinja2
@@ -1,7 +1,10 @@
# Copyright 2026 Marimo. All rights reserved.
# AUTO-GENERATED FILE — DO NOT EDIT.
# Generated from protocols/host/openapi.yaml.
-# Regenerate with `make host-protocol-codegen`.
+{% if features -%}
+# Experimental features: {{ features | join(", ") }}
+{% endif -%}
+# Regenerate with `{{ command }}`.
"""Typed envelope adapters. Payload ownership stays with the caller."""
from __future__ import annotations
@@ -10,8 +13,11 @@ from typing import TYPE_CHECKING, TypeAlias, overload
from msgspec import UNSET, UnsetType
+{% if relative_import -%}
+from . import protocol # noqa: TID252 - generated bindings share a package
+{% else -%}
from marimo._core import protocol
-
+{% endif %}
if TYPE_CHECKING:
from datetime import datetime
diff --git a/pyproject.toml b/pyproject.toml
index 7b961c7599..6d0a091eea 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -173,6 +173,8 @@ test = [
"dirty-equals~=0.9.0",
# For protocol code generation tests
"jinja2>=3.1.6",
+ "openapi-spec-validator>=0.9.0",
+ "python-jsonpath==2.2.1",
# No cp315 wheels as of 2026-10 for matplotlib 3.10 and zstandard; drop the
# python_version markers when published. numpy is listed on its own because
# tests used to get it only through matplotlib.
diff --git a/tests/_core/test_protocol.py b/tests/_core/test_protocol.py
index 8ec91090ce..d451b3c11c 100644
--- a/tests/_core/test_protocol.py
+++ b/tests/_core/test_protocol.py
@@ -17,6 +17,10 @@ from marimo._core import protocol, protocol_events
FIXTURES = (
Path(__file__).resolve().parents[2] / "protocols" / "host" / "fixtures"
)
+EXPERIMENTAL_FIXTURES = (
+ FIXTURES.parent / "experimental" / "analysis" / "fixtures"
+)
+
MODELS = {
"record.json": protocol.HostRecord,
"cell.json": protocol.Cell,
@@ -137,7 +141,9 @@ def round_trip(model: type, text: str) -> None:
@pytest.mark.parametrize(
- "path", sorted(FIXTURES.rglob("*.json")), ids=lambda path: path.name
+ "path",
+ sorted([*FIXTURES.rglob("*.json"), *EXPERIMENTAL_FIXTURES.glob("*.json")]),
+ ids=lambda path: path.name,
)
def test_json_models(path: Path) -> None:
model = (
@@ -149,7 +155,9 @@ def test_json_models(path: Path) -> None:
@pytest.mark.parametrize(
- "path", sorted(FIXTURES.glob("*.sse")), ids=lambda path: path.name
+ "path",
+ sorted([*FIXTURES.glob("*.sse"), *EXPERIMENTAL_FIXTURES.glob("*.sse")]),
+ ids=lambda path: path.name,
)
def test_stream_models(path: Path) -> None:
events = stream_events(path)
@@ -173,7 +181,9 @@ def test_stream_models(path: Path) -> None:
@pytest.mark.parametrize(
- "path", sorted(FIXTURES.glob("*.sse")), ids=lambda path: path.name
+ "path",
+ sorted([*FIXTURES.glob("*.sse"), *EXPERIMENTAL_FIXTURES.glob("*.sse")]),
+ ids=lambda path: path.name,
)
def test_produced_events_match_wire_fixtures(path: Path) -> None:
for frame in stream_events(path):
diff --git a/tests/_lint/test_host_protocol_codegen.py b/tests/_lint/test_host_protocol_codegen.py
index ad56a89f77..4519105b05 100644
--- a/tests/_lint/test_host_protocol_codegen.py
+++ b/tests/_lint/test_host_protocol_codegen.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import importlib
from datetime import datetime
from pathlib import Path
@@ -102,3 +103,26 @@ def test_event_adapters_follow_schema_references(
assert namespace["removed"](protocol.Project, project.id) == (
protocol.ProjectRemovedEvent(protocol.RemovedMessage(id="p1"))
)
+
+
+def test_event_adapters_use_adjacent_models(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ package = tmp_path / "generated_protocol"
+ package.mkdir()
+ (package / "protocol.py").write_bytes(Path(protocol.__file__).read_bytes())
+ document = msgspec.yaml.decode(
+ Path("protocols/host/openapi.yaml").read_bytes()
+ )
+ (package / "protocol_events.py").write_text(
+ event_adapters(document, relative_import=True), encoding="utf-8"
+ )
+ monkeypatch.syspath_prepend(str(tmp_path))
+ models = importlib.import_module("generated_protocol.protocol")
+ events = importlib.import_module("generated_protocol.protocol_events")
+ cell = msgspec.json.decode(
+ Path("protocols/host/fixtures/cell.json").read_bytes(),
+ type=models.Cell,
+ )
+ assert events.body(events.record(cell)) is cell
+ assert events.protocol is models
diff --git a/tests/_lint/test_host_protocol_overlays.py b/tests/_lint/test_host_protocol_overlays.py
new file mode 100644
index 0000000000..f69ac20b5e
--- /dev/null
+++ b/tests/_lint/test_host_protocol_overlays.py
@@ -0,0 +1,126 @@
+# Copyright 2026 Marimo. All rights reserved.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from urllib.parse import unquote
+
+import jsonpath
+import pytest
+from openapi_spec_validator import validate
+
+from protocols.host.generate import ROOT, compose
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+
+ "selection",
+ [
+ None,
+ "all",
+ *(
+ path.parent.name
+ for path in sorted(ROOT.glob("experimental/*/overlay.yaml"))
+ ),
+ ],
+)
+def test_selected_contracts_are_valid(selection: str | None) -> None:
+ document = compose(selection)
+ validate(document)
+ # OpenAPI validation does not resolve every schema reference, including
+ # those in additionalProperties. Check each feature without its siblings.
+ for ref in jsonpath.findall("$..['$ref']", document, strict=True):
+ assert isinstance(ref, str)
+ assert ref.startswith("#/")
+ jsonpath.pointer.resolve(unquote(ref[1:]), document)
+
+
+def test_analysis_selection_preserves_stable_contract() -> None:
+ base = compose()
+ combined = compose("all")
+ assert combined == compose("analysis") == compose("analysis,analysis")
+ assert combined["x-marimo-experimental"] == ["analysis"]
+ assert combined["paths"] == base["paths"]
+ schemas = combined["components"]["schemas"]
+ original = base["components"]["schemas"]
+ added = {"Analysis", "CellAnalysis", "AnalysisEvent", "AnalysisMessage"}
+ assert schemas.keys() - original.keys() == added
+ for name, schema in original.items():
+ if name not in {"ObjectType", "Event"}:
+ assert schemas[name] == schema
+ assert schemas["ObjectType"]["enum"] == [
+ *original["ObjectType"]["enum"],
+ "analysis",
+ ]
+ assert schemas["Event"]["oneOf"] == [
+ *original["Event"]["oneOf"],
+ {"$ref": "#/components/schemas/AnalysisEvent"},
+ ]
+ assert schemas["Event"]["discriminator"]["mapping"] == {
+ **original["Event"]["discriminator"]["mapping"],
+ "analysis": "#/components/schemas/AnalysisEvent",
+ }
+ for name in added:
+ assert schemas[name]["description"].startswith(
+ "Experimental (analysis)."
+ )
+ assert compose() == base
+
+
+ "selection", ["", "unknown", "analysis,unknown", "../analysis"]
+)
+def test_unknown_features_fail(selection: str) -> None:
+ with pytest.raises(ValueError, match="Unknown experimental"):
+ compose(selection)
+
+
+def test_overlay_can_add_an_endpoint(tmp_path: Path) -> None:
+ (tmp_path / "openapi.yaml").write_bytes(
+ (ROOT / "openapi.yaml").read_bytes()
+ )
+ directory = tmp_path / "experimental" / "search"
+ directory.mkdir(parents=True)
+ (directory / "overlay.yaml").write_text(
+ """\
+overlay: 1.0.0
+info:
+ title: Search
+ version: 0.1.0
+actions:
+ - target: $
+ update:
+ paths:
+ /search:
+ get:
+ operationId: search
+ responses:
+ '200':
+ description: Search results
+ content:
+ application/json:
+ schema: { $ref: '#/components/schemas/SearchResults' }
+ components:
+ schemas:
+ SearchResults:
+ type: object
+ properties:
+ text: { type: string }
+ - target: $.paths['/search'].get
+ update:
+ summary: Search notebooks
+""",
+ encoding="utf-8",
+ )
+ base = compose(root=tmp_path)
+ document = compose("search", root=tmp_path)
+ validate(document)
+ assert {path: document["paths"][path] for path in base["paths"]} == base[
+ "paths"
+ ]
+ assert document["paths"]["/search"]["get"]["summary"] == "Search notebooks"
+ assert document["components"]["schemas"]["SearchResults"][
+ "properties"
+ ] == {"text": {"type": "string"}}