Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Allow hosts to withhold cell source (#11361)

Trevor Manzf90784d67d57
diff --git a/frontend/src/core/host/__tests__/protocol.test.ts b/frontend/src/core/host/__tests__/protocol.test.ts
index a637f4f0c6..d166d099f3 100644
--- a/frontend/src/core/host/__tests__/protocol.test.ts
+++ b/frontend/src/core/host/__tests__/protocol.test.ts
@@ -19,6 +19,7 @@ function readFixture(name: string): unknown {
const records = {
"record.json": Protocol.HostRecord,
"cell.json": Protocol.Cell,
+ "cell-withheld.json": Protocol.Cell,
"analysis.json": Protocol.Analysis,
"diagnostic.json": Protocol.Error,
"edit.json": Protocol.EditRequest,
@@ -111,6 +112,20 @@ describe("host protocol bindings", () => {
expect(Protocol.Document.parse(configured)).toEqual(configured);
});
+ it("requires source in edits while allowing withheld source in records", () => {
+ expectTypeOf<Protocol.Cell["code"]>().toEqualTypeOf<string | null>();
+ expectTypeOf<Protocol.CellInput["code"]>().toEqualTypeOf<string>();
+ const cell = Protocol.Cell.parse(readFixture("cell.json"));
+ const { code, ...missing } = cell;
+ expect(Protocol.Cell.safeParse(missing).success).toBe(false);
+ expect(Protocol.Cell.parse({ ...cell, code: "" }).code).toBe("");
+ for (const input of [{ id: cell.id }, { id: cell.id, code: null }]) {
+ expect(Protocol.CellInput.safeParse(input).success).toBe(false);
+ }
+ expect(Protocol.CellInput.parse({ id: cell.id, code }).code).toBe(code);
+ expect(Protocol.CellInput.parse({ id: cell.id, code: "" }).code).toBe("");
+ });
+
it("applies the defaults declared for cell inputs", () => {
expect(
Protocol.CellInput.parse({ id: "c3", code: "x = 1", config: {} }),
diff --git a/frontend/src/core/host/protocol.ts b/frontend/src/core/host/protocol.ts
index f2d643f585..0ca36bd94b 100644
--- a/frontend/src/core/host/protocol.ts
+++ b/frontend/src/core/host/protocol.ts
@@ -618,7 +618,7 @@ export const Execution = zod
.string()
.nullable()
.describe(
- "The submitted code for an execution with no cell. Null for a cell's execution, since the cell holds its code.",
+ "The submitted code for an execution with no cell. Null for a cell's execution; its source is available through the cell when the host exposes it.",
),
code_hash: Etag.describe(
"The `code_hash` of the code that was evaluated, computed as a cell's is.",
@@ -715,7 +715,9 @@ export const CellConfig = zod
disabled: zod.boolean().describe("Whether the cell is kept from running."),
hide_code: zod
.boolean()
- .describe("Whether clients hide the code by default."),
+ .describe(
+ "Whether clients hide the code by default. This is a presentation preference; source availability is represented by `Cell.code`.",
+ ),
expand_output: zod
.boolean()
.describe(
@@ -771,8 +773,9 @@ export const Cell = zod
notebook_id: NotebookId,
code: zod
.string()
+ .nullable()
.describe(
- "The cell's source code, with newline line endings. A host normalizes other line endings when it loads a file.",
+ "The cell's source code, with newline line endings, or null when the host does not expose source to this client. An empty string is an empty cell. A host normalizes other line endings when it loads a file. Source availability does not grant editing permission; see the notebook's operations.",
),
name: zod
.string()
@@ -784,7 +787,7 @@ export const Cell = zod
"Identifies code, name, and config together, and is the precondition for editing the cell. Its text is `code_hash`, `name`, and the config values `column`, `disabled`, `hide_code`, and `expand_output` in that order, each written as a JSON scalar such as `null`, `0`, or `true`.",
),
code_hash: Etag.describe(
- "Identifies the code alone. Its text is `code`. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code.",
+ "Identifies the code alone. Its text is the actual source, even when `code` is null. Withholding source does not change this hash, the cell or document etags, or their versions. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code without needing the source.",
),
version: zod
.int()
@@ -1800,7 +1803,7 @@ export const CellInput = zod
config: CellConfigInput.optional(),
})
.describe(
- "A cell as a client writes it. The host derives the etags and the analysis.",
+ "A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags and the analysis.",
);
export type CellInput = zod.input<typeof CellInput>;
diff --git a/marimo/_core/document.py b/marimo/_core/document.py
index e53271731a..51b1e7108a 100644
--- a/marimo/_core/document.py
+++ b/marimo/_core/document.py
@@ -182,7 +182,7 @@ class Document:
) -> Document:
"""Rebuild a document from the records the notebook published.
- Cell identities, order, and revisions must agree.
+ Source must be available; cell identities, order, and revisions must agree.
"""
by_id = {cell.id: cell for cell in cells}
if len(by_id) != len(cells) or (
@@ -194,13 +194,19 @@ class Document:
)
if any(cell.notebook_id != document.notebook_id for cell in cells):
raise ValueError("cells must belong to the document")
- restored = cls(
- [
- DocumentCell(cell.id, cell.code, cell.name, cell.config)
- for cell in (by_id[cell_id] for cell_id in document.order)
- ],
- document.app_config,
- )
+ restored_cells = []
+ for cell_id in document.order:
+ record = by_id[cell_id]
+ if record.code is None:
+ raise ValueError(
+ "cannot restore a document with withheld source"
+ )
+ restored_cells.append(
+ DocumentCell(
+ record.id, record.code, record.name, record.config
+ )
+ )
+ restored = cls(restored_cells, document.app_config)
for cell in restored:
original = by_id[cell.id]
if (
diff --git a/marimo/_core/protocol.py b/marimo/_core/protocol.py
index b3a8fcc8e1..ba2bc97dca 100644
--- a/marimo/_core/protocol.py
+++ b/marimo/_core/protocol.py
@@ -47,7 +47,10 @@ class CellConfig(Struct, frozen=True):
),
]
hide_code: Annotated[
- bool, Meta(description="Whether clients hide the code by default.")
+ bool,
+ Meta(
+ description="Whether clients hide the code by default. This is a presentation preference; source availability is represented by `Cell.code`."
+ ),
]
@@ -758,16 +761,19 @@ class AttachmentRemovedEvent(
class Cell(Struct, frozen=True):
- code: Annotated[
- str,
- Meta(
- description="The cell's source code, with newline line endings. A host normalizes other line endings when it loads a file."
- ),
- ]
+ code: (
+ Annotated[
+ str,
+ Meta(
+ description="The cell's source code, with newline line endings, or null when the host does not expose source to this client. An empty string is an empty cell. A host normalizes other line endings when it loads a file. Source availability does not grant editing permission; see the notebook's operations."
+ ),
+ ]
+ | None
+ )
code_hash: Annotated[
Etag,
Meta(
- description="Identifies the code alone. Its text is `code`. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code."
+ description="Identifies the code alone. Its text is the actual source, even when `code` is null. Withholding source does not change this hash, the cell or document etags, or their versions. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code without needing the source."
),
]
config: CellConfig
@@ -1685,7 +1691,7 @@ class Execution(Struct, frozen=True):
Annotated[
str,
Meta(
- description="The submitted code for an execution with no cell. Null for a cell's execution, since the cell holds its code."
+ description="The submitted code for an execution with no cell. Null for a cell's execution; its source is available through the cell when the host exposes it."
),
]
| None
diff --git a/protocols/host/conformance/contract.py b/protocols/host/conformance/contract.py
index 116482ce7d..bb2d9b755b 100644
--- a/protocols/host/conformance/contract.py
+++ b/protocols/host/conformance/contract.py
@@ -206,7 +206,8 @@ def document_state(
assert sorted(document["order"]) == sorted(cells)
for cell in cells.values():
assert cell["notebook_id"] == document["notebook_id"]
- assert cell["code_hash"] == etag(cell["code"])
+ if cell["code"] is not None:
+ assert cell["code_hash"] == etag(cell["code"])
config = cell["config"]
parts = [cell["code_hash"], cell["name"]]
parts.extend(
diff --git a/protocols/host/conformance/test_contract.py b/protocols/host/conformance/test_contract.py
index f8c996d06b..27eb82b78f 100644
--- a/protocols/host/conformance/test_contract.py
+++ b/protocols/host/conformance/test_contract.py
@@ -25,6 +25,7 @@ FIXTURES = ROOT / "fixtures"
MODELS = {
"record.json": "HostRecord",
"cell.json": "Cell",
+ "cell-withheld.json": "Cell",
"analysis.json": "Analysis",
"diagnostic.json": "Error",
"edit.json": "EditRequest",
@@ -98,6 +99,45 @@ def test_required_nullable_fields_and_future_fields():
validate("Execution", {**value, field: ""})
+def test_cell_source_is_required_but_nullable_only_in_records():
+ cell = json.loads((FIXTURES / "cell.json").read_text())
+ for code in (None, "", cell["code"]):
+ validate("Cell", {**cell, "code": code})
+ missing = {key: value for key, value in cell.items() if key != "code"}
+ with pytest.raises(ValidationError):
+ validate("Cell", missing)
+ for value in ({"id": cell["id"]}, {"id": cell["id"], "code": None}):
+ with pytest.raises(ValidationError):
+ validate("CellInput", value)
+ validate("CellInput", {"id": cell["id"], "code": ""})
+
+
+async def test_document_checker_accepts_withheld_source_and_checks_etags():
+ snapshot = []
+ async for frame in frames(
+ lines((FIXTURES / "notebook-events.sse").read_text())
+ ):
+ if frame.event == "ready":
+ break
+ snapshot.append(frame)
+ document, cells = document_state(snapshot)
+ for frame in snapshot:
+ if frame.event == "cell":
+ message = frame.message()
+ message["cell"]["code"] = None
+ frame.data = json.dumps(message)
+ assert document_state(snapshot) == (
+ document,
+ {key: {**cell, "code": None} for key, cell in cells.items()},
+ )
+ cell = next(frame for frame in snapshot if frame.event == "cell")
+ message = cell.message()
+ message["cell"]["code_hash"] = "0000000000000000"
+ cell.data = json.dumps(message)
+ with pytest.raises(AssertionError):
+ document_state(snapshot)
+
+
async def test_document_checker_rejects_wrong_hashes_and_ordering():
snapshot = []
async for frame in frames(
diff --git a/protocols/host/fixtures/README.md b/protocols/host/fixtures/README.md
index 4d86e6b4af..de855ba5d7 100644
--- a/protocols/host/fixtures/README.md
+++ b/protocols/host/fixtures/README.md
@@ -9,7 +9,9 @@ opaque. Gaps represent events outside an excerpt or its filter.
shown in `run.sse`. Its snapshot includes the display and console output
already produced. Both streams end with the same execution and run completion
events, using the same cursors. `cell.json` is cell `c3` from that snapshot,
-and `analysis.json` is the analysis of that document.
+and `analysis.json` is the analysis of that document. `cell-withheld.json`
+is the same cell with source withheld from the client: its hashes and version
+are unchanged.
`run-cells-request.json` requests `c3`, which defines `x`. In `run-cells.sse`,
the runtime then evaluates `k7f2`, which defines `y` using `x`. The dependent
diff --git a/protocols/host/fixtures/cell-withheld.json b/protocols/host/fixtures/cell-withheld.json
new file mode 100644
index 0000000000..8a24b35009
--- /dev/null
+++ b/protocols/host/fixtures/cell-withheld.json
@@ -0,0 +1,15 @@
+{
+ "id": "c3",
+ "notebook_id": "n1",
+ "code": null,
+ "name": "_",
+ "config": {
+ "column": null,
+ "disabled": false,
+ "hide_code": false,
+ "expand_output": false
+ },
+ "etag": "5cf78587dcf22d8d",
+ "code_hash": "0f71456db2ea83f4",
+ "version": 1
+}
diff --git a/protocols/host/openapi.yaml b/protocols/host/openapi.yaml
index be131c403b..ff60dafea1 100644
--- a/protocols/host/openapi.yaml
+++ b/protocols/host/openapi.yaml
@@ -1229,7 +1229,7 @@ components:
description: The cell this execution belongs to. Null for scratch code, a cell outside the document.
code:
type: [string, 'null']
- description: The submitted code for an execution with no cell. Null for a cell's execution, since the cell holds its code.
+ description: The submitted code for an execution with no cell. Null for a cell's execution; its source is available through the cell when the host exposes it.
code_hash:
$ref: '#/components/schemas/Etag'
description: The `code_hash` of the code that was evaluated, computed as a cell's is.
@@ -1314,7 +1314,7 @@ components:
description: Whether the cell is kept from running.
hide_code:
type: boolean
- description: Whether clients hide the code by default.
+ description: Whether clients hide the code by default. This is a presentation preference; source availability is represented by `Cell.code`.
expand_output:
type: boolean
description: Whether clients show the full output rather than a scrolled box.
@@ -1370,8 +1370,8 @@ components:
id: { $ref: '#/components/schemas/CellId' }
notebook_id: { $ref: '#/components/schemas/NotebookId' }
code:
- type: string
- description: The cell's source code, with newline line endings. A host normalizes other line endings when it loads a file.
+ type: [string, 'null']
+ description: The cell's source code, with newline line endings, or null when the host does not expose source to this client. An empty string is an empty cell. A host normalizes other line endings when it loads a file. Source availability does not grant editing permission; see the notebook's operations.
name:
type: string
description: The cell's name. `_` for an anonymous cell; otherwise a Python identifier, unique within the notebook.
@@ -1381,7 +1381,7 @@ components:
description: Identifies code, name, and config together, and is the precondition for editing the cell. Its text is `code_hash`, `name`, and the config values `column`, `disabled`, `hide_code`, and `expand_output` in that order, each written as a JSON scalar such as `null`, `0`, or `true`.
code_hash:
$ref: '#/components/schemas/Etag'
- description: Identifies the code alone. Its text is `code`. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code.
+ description: Identifies the code alone. Its text is the actual source, even when `code` is null. Withholding source does not change this hash, the cell or document etags, or their versions. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code without needing the source.
version:
type: integer
minimum: 1
@@ -1966,7 +1966,7 @@ components:
default: false
CellInput:
type: object
- description: A cell as a client writes it. The host derives the etags and the analysis.
+ description: A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags and the analysis.
required: [id, code]
properties:
id:
diff --git a/protocols/host/spec.md b/protocols/host/spec.md
index c355c9cbac..f1c758673e 100644
--- a/protocols/host/spec.md
+++ b/protocols/host/spec.md
@@ -83,6 +83,19 @@ the document version at which the cell was created or last changed, so of two
records of a cell, the higher version is the newer. Versions are comparable
only within one notebook.
+A host MAY withhold a cell's source from a client by publishing `code: null`.
+The field MUST remain present; an empty string means an empty cell. The host
+MUST derive `code_hash` from the actual source even when it withholds that
+source. Withholding source MUST NOT change cell or document etags or versions.
+Clients can therefore compare execution hashes and dependency revisions to
+assess staleness without access to source.
+
+Hosts MUST apply their source visibility policy consistently to snapshots,
+live events, replay, and responses containing cells. `hide_code` is only a
+presentation preference; it does not withhold source. A null `code` field does
+not promise that source cannot be inferred from hashes, outputs, diagnostics,
+or other protocol data.
+
A host MAY also publish an **analysis** of the document: each cell's
definitions, references, and problems, so clients can understand a cell's role
in the computation and identify problems before execution. Because cells share
@@ -294,6 +307,11 @@ transient state. For example, a notebook can permit execution while its runtime
is stopped. A client uses the notebook's current state to determine what
preparation is needed before calling an operation.
+Source visibility and editing permission are independent. A client MAY display
+available source without `notebook.edit`. An editor for existing source needs
+both a non-null `code` and permission to edit. Withholding source does not
+remove the notebook's execution records, outputs, or runtime state.
+
### Requests and decisions
Requests can produce changes after their HTTP response has arrived. The host
@@ -317,6 +335,11 @@ Edits describe a group of changes to the document. Hosts MUST apply each group
atomically and reject the whole group if its preconditions are stale. Clients
can inspect the conflicting state and decide how to revise their edit.
+`put_cell` replaces the whole cell and MUST include string-valued source in
+`CellInput.code`. A client MUST NOT round-trip a withheld `code: null` as an
+edit or substitute an empty string for it. Other changes, such as reordering
+or deleting cells, do not require source but still require editing permission.
+
Preconditions use **etags**, identifiers derived from content. They let clients
edit different parts of a document independently while detecting competing
changes to the same part. For example, two clients can update different cells,
diff --git a/tests/_core/test_notebook_document.py b/tests/_core/test_notebook_document.py
index 405963e9f4..2dcb4ff6d2 100644
--- a/tests/_core/test_notebook_document.py
+++ b/tests/_core/test_notebook_document.py
@@ -378,6 +378,13 @@ class TestFromRecords:
assert restored == notebook.document
assert document.saved_etag != restored.etag
+ def test_rejects_withheld_source(self) -> None:
+ document, cells = records(hosted(Document(fixture_cells(), {})))
+ cells[0] = replace(cells[0], code=None)
+
+ with pytest.raises(ValueError, match="withheld source"):
+ Document.restore(document, cells)
+
@pytest.mark.parametrize(
"invalid", ["order", "notebook-id", "cell-etag", "document-etag"]
)
diff --git a/tests/_core/test_protocol.py b/tests/_core/test_protocol.py
index fa6cc22ab0..8ec91090ce 100644
--- a/tests/_core/test_protocol.py
+++ b/tests/_core/test_protocol.py
@@ -20,6 +20,7 @@ FIXTURES = (
MODELS = {
"record.json": protocol.HostRecord,
"cell.json": protocol.Cell,
+ "cell-withheld.json": protocol.Cell,
"analysis.json": protocol.Analysis,
"diagnostic.json": protocol.Error,
"edit.json": protocol.EditRequest,
@@ -611,3 +612,24 @@ def test_required_nullable_fields(field: str) -> None:
msgspec.json.decode(
json.dumps({**wire, field: ""}), type=protocol.Execution
)
+
+
+def test_cell_source_is_required_but_nullable_only_in_records() -> None:
+ cell = json.loads((FIXTURES / "cell.json").read_text(encoding="utf-8"))
+ for code in (None, "", cell["code"]):
+ value = msgspec.json.decode(
+ json.dumps({**cell, "code": code}), type=protocol.Cell
+ )
+ assert value.code == code
+ missing = {key: value for key, value in cell.items() if key != "code"}
+ with pytest.raises(
+ msgspec.ValidationError, match="missing required field"
+ ):
+ msgspec.json.decode(json.dumps(missing), type=protocol.Cell)
+ for value in ({"id": cell["id"]}, {"id": cell["id"], "code": None}):
+ with pytest.raises(msgspec.ValidationError):
+ msgspec.json.decode(json.dumps(value), type=protocol.CellInput)
+ value = msgspec.json.decode(
+ json.dumps({"id": cell["id"], "code": ""}), type=protocol.CellInput
+ )
+ assert value.code == ""