A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Allow hosts to withhold cell source (#11361)
Trevor Manzf90784d67d57
diff --git a/frontend/src/core/host/__tests__/protocol.test.ts b/frontend/src/core/host/__tests__/protocol.test.tsindex a637f4f0c6..d166d099f3 100644--- a/frontend/src/core/host/__tests__/protocol.test.ts+++ b/frontend/src/core/host/__tests__/protocol.test.ts@@ -19,6 +19,7 @@ function readFixture(name: string): unknown {const records = {"record.json": Protocol.HostRecord,"cell.json": Protocol.Cell,+ "cell-withheld.json": Protocol.Cell,"analysis.json": Protocol.Analysis,"diagnostic.json": Protocol.Error,"edit.json": Protocol.EditRequest,@@ -111,6 +112,20 @@ describe("host protocol bindings", () => {expect(Protocol.Document.parse(configured)).toEqual(configured);});+ it("requires source in edits while allowing withheld source in records", () => {+ expectTypeOf<Protocol.Cell["code"]>().toEqualTypeOf<string | null>();+ expectTypeOf<Protocol.CellInput["code"]>().toEqualTypeOf<string>();+ const cell = Protocol.Cell.parse(readFixture("cell.json"));+ const { code, ...missing } = cell;+ expect(Protocol.Cell.safeParse(missing).success).toBe(false);+ expect(Protocol.Cell.parse({ ...cell, code: "" }).code).toBe("");+ for (const input of [{ id: cell.id }, { id: cell.id, code: null }]) {+ expect(Protocol.CellInput.safeParse(input).success).toBe(false);+ }+ expect(Protocol.CellInput.parse({ id: cell.id, code }).code).toBe(code);+ expect(Protocol.CellInput.parse({ id: cell.id, code: "" }).code).toBe("");+ });+it("applies the defaults declared for cell inputs", () => {expect(Protocol.CellInput.parse({ id: "c3", code: "x = 1", config: {} }),diff --git a/frontend/src/core/host/protocol.ts b/frontend/src/core/host/protocol.tsindex f2d643f585..0ca36bd94b 100644--- a/frontend/src/core/host/protocol.ts+++ b/frontend/src/core/host/protocol.ts@@ -618,7 +618,7 @@ export const Execution = zod.string().nullable().describe(- "The submitted code for an execution with no cell. Null for a cell's execution, since the cell holds its code.",+ "The submitted code for an execution with no cell. Null for a cell's execution; its source is available through the cell when the host exposes it.",),code_hash: Etag.describe("The `code_hash` of the code that was evaluated, computed as a cell's is.",@@ -715,7 +715,9 @@ export const CellConfig = zoddisabled: zod.boolean().describe("Whether the cell is kept from running."),hide_code: zod.boolean()- .describe("Whether clients hide the code by default."),+ .describe(+ "Whether clients hide the code by default. This is a presentation preference; source availability is represented by `Cell.code`.",+ ),expand_output: zod.boolean().describe(@@ -771,8 +773,9 @@ export const Cell = zodnotebook_id: NotebookId,code: zod.string()+ .nullable().describe(- "The cell's source code, with newline line endings. A host normalizes other line endings when it loads a file.",+ "The cell's source code, with newline line endings, or null when the host does not expose source to this client. An empty string is an empty cell. A host normalizes other line endings when it loads a file. Source availability does not grant editing permission; see the notebook's operations.",),name: zod.string()@@ -784,7 +787,7 @@ export const Cell = zod"Identifies code, name, and config together, and is the precondition for editing the cell. Its text is `code_hash`, `name`, and the config values `column`, `disabled`, `hide_code`, and `expand_output` in that order, each written as a JSON scalar such as `null`, `0`, or `true`.",),code_hash: Etag.describe(- "Identifies the code alone. Its text is `code`. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code.",+ "Identifies the code alone. Its text is the actual source, even when `code` is null. Withholding source does not change this hash, the cell or document etags, or their versions. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code without needing the source.",),version: zod.int()@@ -1800,7 +1803,7 @@ export const CellInput = zodconfig: CellConfigInput.optional(),}).describe(- "A cell as a client writes it. The host derives the etags and the analysis.",+ "A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags and the analysis.",);export type CellInput = zod.input<typeof CellInput>;diff --git a/marimo/_core/document.py b/marimo/_core/document.pyindex e53271731a..51b1e7108a 100644--- a/marimo/_core/document.py+++ b/marimo/_core/document.py@@ -182,7 +182,7 @@ class Document:) -> Document:"""Rebuild a document from the records the notebook published.- Cell identities, order, and revisions must agree.+ Source must be available; cell identities, order, and revisions must agree."""by_id = {cell.id: cell for cell in cells}if len(by_id) != len(cells) or (@@ -194,13 +194,19 @@ class Document:)if any(cell.notebook_id != document.notebook_id for cell in cells):raise ValueError("cells must belong to the document")- restored = cls(- [- DocumentCell(cell.id, cell.code, cell.name, cell.config)- for cell in (by_id[cell_id] for cell_id in document.order)- ],- document.app_config,- )+ restored_cells = []+ for cell_id in document.order:+ record = by_id[cell_id]+ if record.code is None:+ raise ValueError(+ "cannot restore a document with withheld source"+ )+ restored_cells.append(+ DocumentCell(+ record.id, record.code, record.name, record.config+ )+ )+ restored = cls(restored_cells, document.app_config)for cell in restored:original = by_id[cell.id]if (diff --git a/marimo/_core/protocol.py b/marimo/_core/protocol.pyindex b3a8fcc8e1..ba2bc97dca 100644--- a/marimo/_core/protocol.py+++ b/marimo/_core/protocol.py@@ -47,7 +47,10 @@ class CellConfig(Struct, frozen=True):),]hide_code: Annotated[- bool, Meta(description="Whether clients hide the code by default.")+ bool,+ Meta(+ description="Whether clients hide the code by default. This is a presentation preference; source availability is represented by `Cell.code`."+ ),]@@ -758,16 +761,19 @@ class AttachmentRemovedEvent(class Cell(Struct, frozen=True):- code: Annotated[- str,- Meta(- description="The cell's source code, with newline line endings. A host normalizes other line endings when it loads a file."- ),- ]+ code: (+ Annotated[+ str,+ Meta(+ description="The cell's source code, with newline line endings, or null when the host does not expose source to this client. An empty string is an empty cell. A host normalizes other line endings when it loads a file. Source availability does not grant editing permission; see the notebook's operations."+ ),+ ]+ | None+ )code_hash: Annotated[Etag,Meta(- description="Identifies the code alone. Its text is `code`. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code."+ description="Identifies the code alone. Its text is the actual source, even when `code` is null. Withholding source does not change this hash, the cell or document etags, or their versions. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code without needing the source."),]config: CellConfig@@ -1685,7 +1691,7 @@ class Execution(Struct, frozen=True):Annotated[str,Meta(- description="The submitted code for an execution with no cell. Null for a cell's execution, since the cell holds its code."+ description="The submitted code for an execution with no cell. Null for a cell's execution; its source is available through the cell when the host exposes it."),]| Nonediff --git a/protocols/host/conformance/contract.py b/protocols/host/conformance/contract.pyindex 116482ce7d..bb2d9b755b 100644--- a/protocols/host/conformance/contract.py+++ b/protocols/host/conformance/contract.py@@ -206,7 +206,8 @@ def document_state(assert sorted(document["order"]) == sorted(cells)for cell in cells.values():assert cell["notebook_id"] == document["notebook_id"]- assert cell["code_hash"] == etag(cell["code"])+ if cell["code"] is not None:+ assert cell["code_hash"] == etag(cell["code"])config = cell["config"]parts = [cell["code_hash"], cell["name"]]parts.extend(diff --git a/protocols/host/conformance/test_contract.py b/protocols/host/conformance/test_contract.pyindex f8c996d06b..27eb82b78f 100644--- a/protocols/host/conformance/test_contract.py+++ b/protocols/host/conformance/test_contract.py@@ -25,6 +25,7 @@ FIXTURES = ROOT / "fixtures"MODELS = {"record.json": "HostRecord","cell.json": "Cell",+ "cell-withheld.json": "Cell","analysis.json": "Analysis","diagnostic.json": "Error","edit.json": "EditRequest",@@ -98,6 +99,45 @@ def test_required_nullable_fields_and_future_fields():validate("Execution", {**value, field: ""})+def test_cell_source_is_required_but_nullable_only_in_records():+ cell = json.loads((FIXTURES / "cell.json").read_text())+ for code in (None, "", cell["code"]):+ validate("Cell", {**cell, "code": code})+ missing = {key: value for key, value in cell.items() if key != "code"}+ with pytest.raises(ValidationError):+ validate("Cell", missing)+ for value in ({"id": cell["id"]}, {"id": cell["id"], "code": None}):+ with pytest.raises(ValidationError):+ validate("CellInput", value)+ validate("CellInput", {"id": cell["id"], "code": ""})+++async def test_document_checker_accepts_withheld_source_and_checks_etags():+ snapshot = []+ async for frame in frames(+ lines((FIXTURES / "notebook-events.sse").read_text())+ ):+ if frame.event == "ready":+ break+ snapshot.append(frame)+ document, cells = document_state(snapshot)+ for frame in snapshot:+ if frame.event == "cell":+ message = frame.message()+ message["cell"]["code"] = None+ frame.data = json.dumps(message)+ assert document_state(snapshot) == (+ document,+ {key: {**cell, "code": None} for key, cell in cells.items()},+ )+ cell = next(frame for frame in snapshot if frame.event == "cell")+ message = cell.message()+ message["cell"]["code_hash"] = "0000000000000000"+ cell.data = json.dumps(message)+ with pytest.raises(AssertionError):+ document_state(snapshot)++async def test_document_checker_rejects_wrong_hashes_and_ordering():snapshot = []async for frame in frames(diff --git a/protocols/host/fixtures/README.md b/protocols/host/fixtures/README.mdindex 4d86e6b4af..de855ba5d7 100644--- a/protocols/host/fixtures/README.md+++ b/protocols/host/fixtures/README.md@@ -9,7 +9,9 @@ opaque. Gaps represent events outside an excerpt or its filter.shown in `run.sse`. Its snapshot includes the display and console outputalready produced. Both streams end with the same execution and run completionevents, using the same cursors. `cell.json` is cell `c3` from that snapshot,-and `analysis.json` is the analysis of that document.+and `analysis.json` is the analysis of that document. `cell-withheld.json`+is the same cell with source withheld from the client: its hashes and version+are unchanged.`run-cells-request.json` requests `c3`, which defines `x`. In `run-cells.sse`,the runtime then evaluates `k7f2`, which defines `y` using `x`. The dependentdiff --git a/protocols/host/fixtures/cell-withheld.json b/protocols/host/fixtures/cell-withheld.jsonnew file mode 100644index 0000000000..8a24b35009--- /dev/null+++ b/protocols/host/fixtures/cell-withheld.json@@ -0,0 +1,15 @@+{+ "id": "c3",+ "notebook_id": "n1",+ "code": null,+ "name": "_",+ "config": {+ "column": null,+ "disabled": false,+ "hide_code": false,+ "expand_output": false+ },+ "etag": "5cf78587dcf22d8d",+ "code_hash": "0f71456db2ea83f4",+ "version": 1+}diff --git a/protocols/host/openapi.yaml b/protocols/host/openapi.yamlindex be131c403b..ff60dafea1 100644--- a/protocols/host/openapi.yaml+++ b/protocols/host/openapi.yaml@@ -1229,7 +1229,7 @@ components:description: The cell this execution belongs to. Null for scratch code, a cell outside the document.code:type: [string, 'null']- description: The submitted code for an execution with no cell. Null for a cell's execution, since the cell holds its code.+ description: The submitted code for an execution with no cell. Null for a cell's execution; its source is available through the cell when the host exposes it.code_hash:$ref: '#/components/schemas/Etag'description: The `code_hash` of the code that was evaluated, computed as a cell's is.@@ -1314,7 +1314,7 @@ components:description: Whether the cell is kept from running.hide_code:type: boolean- description: Whether clients hide the code by default.+ description: Whether clients hide the code by default. This is a presentation preference; source availability is represented by `Cell.code`.expand_output:type: booleandescription: Whether clients show the full output rather than a scrolled box.@@ -1370,8 +1370,8 @@ components:id: { $ref: '#/components/schemas/CellId' }notebook_id: { $ref: '#/components/schemas/NotebookId' }code:- type: string- description: The cell's source code, with newline line endings. A host normalizes other line endings when it loads a file.+ type: [string, 'null']+ description: The cell's source code, with newline line endings, or null when the host does not expose source to this client. An empty string is an empty cell. A host normalizes other line endings when it loads a file. Source availability does not grant editing permission; see the notebook's operations.name:type: stringdescription: The cell's name. `_` for an anonymous cell; otherwise a Python identifier, unique within the notebook.@@ -1381,7 +1381,7 @@ components:description: Identifies code, name, and config together, and is the precondition for editing the cell. Its text is `code_hash`, `name`, and the config values `column`, `disabled`, `hide_code`, and `expand_output` in that order, each written as a JSON scalar such as `null`, `0`, or `true`.code_hash:$ref: '#/components/schemas/Etag'- description: Identifies the code alone. Its text is `code`. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code.+ description: Identifies the code alone. Its text is the actual source, even when `code` is null. Withholding source does not change this hash, the cell or document etags, or their versions. Unchanged by renaming the cell or editing its config. Compare with an execution's `code_hash` to tell whether a result came from the current code without needing the source.version:type: integerminimum: 1@@ -1966,7 +1966,7 @@ components:default: falseCellInput:type: object- description: A cell as a client writes it. The host derives the etags and the analysis.+ description: A cell as a client writes it. Code must be a string, even when the host withholds source in cell records. Null is not an edit or a request to preserve existing source. The host derives the etags and the analysis.required: [id, code]properties:id:diff --git a/protocols/host/spec.md b/protocols/host/spec.mdindex c355c9cbac..f1c758673e 100644--- a/protocols/host/spec.md+++ b/protocols/host/spec.md@@ -83,6 +83,19 @@ the document version at which the cell was created or last changed, so of tworecords of a cell, the higher version is the newer. Versions are comparableonly within one notebook.+A host MAY withhold a cell's source from a client by publishing `code: null`.+The field MUST remain present; an empty string means an empty cell. The host+MUST derive `code_hash` from the actual source even when it withholds that+source. Withholding source MUST NOT change cell or document etags or versions.+Clients can therefore compare execution hashes and dependency revisions to+assess staleness without access to source.++Hosts MUST apply their source visibility policy consistently to snapshots,+live events, replay, and responses containing cells. `hide_code` is only a+presentation preference; it does not withhold source. A null `code` field does+not promise that source cannot be inferred from hashes, outputs, diagnostics,+or other protocol data.+A host MAY also publish an **analysis** of the document: each cell'sdefinitions, references, and problems, so clients can understand a cell's rolein the computation and identify problems before execution. Because cells share@@ -294,6 +307,11 @@ transient state. For example, a notebook can permit execution while its runtimeis stopped. A client uses the notebook's current state to determine whatpreparation is needed before calling an operation.+Source visibility and editing permission are independent. A client MAY display+available source without `notebook.edit`. An editor for existing source needs+both a non-null `code` and permission to edit. Withholding source does not+remove the notebook's execution records, outputs, or runtime state.+### Requests and decisionsRequests can produce changes after their HTTP response has arrived. The host@@ -317,6 +335,11 @@ Edits describe a group of changes to the document. Hosts MUST apply each groupatomically and reject the whole group if its preconditions are stale. Clientscan inspect the conflicting state and decide how to revise their edit.+`put_cell` replaces the whole cell and MUST include string-valued source in+`CellInput.code`. A client MUST NOT round-trip a withheld `code: null` as an+edit or substitute an empty string for it. Other changes, such as reordering+or deleting cells, do not require source but still require editing permission.+Preconditions use **etags**, identifiers derived from content. They let clientsedit different parts of a document independently while detecting competingchanges to the same part. For example, two clients can update different cells,diff --git a/tests/_core/test_notebook_document.py b/tests/_core/test_notebook_document.pyindex 405963e9f4..2dcb4ff6d2 100644--- a/tests/_core/test_notebook_document.py+++ b/tests/_core/test_notebook_document.py@@ -378,6 +378,13 @@ class TestFromRecords:assert restored == notebook.documentassert document.saved_etag != restored.etag+ def test_rejects_withheld_source(self) -> None:+ document, cells = records(hosted(Document(fixture_cells(), {})))+ cells[0] = replace(cells[0], code=None)++ with pytest.raises(ValueError, match="withheld source"):+ Document.restore(document, cells)+@pytest.mark.parametrize("invalid", ["order", "notebook-id", "cell-etag", "document-etag"])diff --git a/tests/_core/test_protocol.py b/tests/_core/test_protocol.pyindex fa6cc22ab0..8ec91090ce 100644--- a/tests/_core/test_protocol.py+++ b/tests/_core/test_protocol.py@@ -20,6 +20,7 @@ FIXTURES = (MODELS = {"record.json": protocol.HostRecord,"cell.json": protocol.Cell,+ "cell-withheld.json": protocol.Cell,"analysis.json": protocol.Analysis,"diagnostic.json": protocol.Error,"edit.json": protocol.EditRequest,@@ -611,3 +612,24 @@ def test_required_nullable_fields(field: str) -> None:msgspec.json.decode(json.dumps({**wire, field: ""}), type=protocol.Execution)+++def test_cell_source_is_required_but_nullable_only_in_records() -> None:+ cell = json.loads((FIXTURES / "cell.json").read_text(encoding="utf-8"))+ for code in (None, "", cell["code"]):+ value = msgspec.json.decode(+ json.dumps({**cell, "code": code}), type=protocol.Cell+ )+ assert value.code == code+ missing = {key: value for key, value in cell.items() if key != "code"}+ with pytest.raises(+ msgspec.ValidationError, match="missing required field"+ ):+ msgspec.json.decode(json.dumps(missing), type=protocol.Cell)+ for value in ({"id": cell["id"]}, {"id": cell["id"], "code": None}):+ with pytest.raises(msgspec.ValidationError):+ msgspec.json.decode(json.dumps(value), type=protocol.CellInput)+ value = msgspec.json.decode(+ json.dumps({"id": cell["id"], "code": ""}), type=protocol.CellInput+ )+ assert value.code == ""