A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Add Node.js host discovery to `@marimo-team/client` (#11401)
Trevor Manz4cfa04b12f83
diff --git a/packages/client/README.md b/packages/client/README.mdindex 29add73451..17ed6557ef 100644--- a/packages/client/README.md+++ b/packages/client/README.md@@ -41,6 +41,27 @@ for await (const event of client.events({ types: ["notebook"] })) {may still be indexing when `ready` arrives, so more notebooks can appearafterward.+## Discover local hosts in Node.js++`discoverHosts()` performs [local discovery](../../protocols/host/spec.md#local-discovery)+on macOS and Linux (requires `procps` on Linux). This example lists the notebooks+each discovered host currently knows about:++```ts+import { Client } from "@marimo-team/client";+import { discoverHosts } from "@marimo-team/client/node";++for await (const host of discoverHosts()) {+ const client = new Client(host.url, { token: host.token });+ for await (const event of client.events({ types: ["notebook"] })) {+ if (event.event === "ready") break;+ if (event.event === "notebook") {+ console.log(event.data.notebook.id, event.data.notebook.name);+ }+ }+}+```+## Work with a notebookUse an id or a notebook record from the catalog to get a `NotebookClient`.diff --git a/packages/client/package.json b/packages/client/package.jsonindex 5c8d6c56e7..d8916cd3e4 100644--- a/packages/client/package.json+++ b/packages/client/package.json@@ -8,6 +8,7 @@"license": "Apache-2.0","exports": {".": "./dist/index.js",+ "./node": "./dist/node.js","./protocol": "./dist/protocol.js"},"files": [diff --git a/packages/client/src/__tests__/discovery.test.ts b/packages/client/src/__tests__/discovery.test.tsnew file mode 100644index 0000000000..779a02aa69--- /dev/null+++ b/packages/client/src/__tests__/discovery.test.ts@@ -0,0 +1,254 @@+/* Copyright 2026 Marimo. All rights reserved. */++import {+ chmod,+ mkdir,+ mkdtemp,+ readdir,+ rm,+ symlink,+ writeFile,+} from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import process from "node:process";+import { promisify } from "node:util";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { Client } from "../index.ts";+import { discoverHosts, type DiscoverHostsOptions } from "../node.ts";+import type { HostRecordOutput } from "../protocol.ts";++const runPs = vi.hoisted(() => vi.fn());+vi.mock("node:child_process", () => ({+ execFile: Object.assign(vi.fn(), {+ [Symbol.for("nodejs.util.promisify.custom")]: runPs,+ }),+}));++const record: HostRecordOutput = {+ version: 1,+ name: "marimo",+ url: "http://127.0.0.1:2718/prefix/api/marimo/v1",+ token: "secret",+ process: { pid: 123, started_at: "2026-10-21T12:34:56.123Z" },+};++describe.skipIf(!["darwin", "linux"].includes(process.platform))(+ "discoverHosts",+ () => {+ let stateDir: string;+ let directory: string;++ beforeEach(async () => {+ stateDir = await mkdtemp(join(tmpdir(), "marimo-discovery-"));+ directory = join(stateDir, "hosts-v1");+ await mkdir(directory, { mode: 0o700 });+ runPs.mockReset();+ runPs.mockResolvedValue({+ stdout: "Wed Oct 21 12:34:56 2026 S+\n",+ stderr: "",+ });+ });++ afterEach(async () => {+ vi.restoreAllMocks();+ vi.unstubAllEnvs();+ await rm(stateDir, { recursive: true, force: true });+ });++ const collect = async (options: DiscoverHostsOptions = { stateDir }) => {+ const records = [];+ for await (const host of discoverHosts(options)) records.push(host);+ return records;+ };++ const write = async (value: unknown = record, name = "host.json") => {+ const path = join(directory, name);+ await writeFile(path, JSON.stringify(value), { mode: 0o600 });+ return path;+ };++ it("yields records usable with the existing Client constructor", async () => {+ await write({ ...record, future_field: true });+ await write(+ { ...record, name: "VS Code", url: "http://[::1]:2719/api/marimo/v1" },+ "other.json",+ );+ const records = await collect();+ expect(records).toHaveLength(2);+ expect(records).toContainEqual(record);+ const fetch = vi.fn(+ async (_input: RequestInfo | URL, _init?: RequestInit) =>+ new Response(null, { status: 204 }),+ );+ const host = records.find((host) => host.name === "marimo")!;+ const client = new Client(host.url, { token: host.token, fetch });+ await client.notebook("n1").delete();+ const request = new Request(...fetch.mock.calls[0]);+ expect(request.url).toBe(`${record.url}/notebooks/n1`);+ expect(request.headers.get("Authorization")).toBe("Bearer secret");+ expect(runPs).toHaveBeenCalledWith(+ "/bin/ps",+ ["-p", "123", "-o", "lstart=", "-o", "stat="],+ expect.objectContaining({+ env: expect.objectContaining({ LC_ALL: "C", TZ: "UTC" }),+ }),+ );+ });++ it("uses the same XDG state directory as the host", async () => {+ vi.stubEnv("XDG_STATE_HOME", stateDir);+ directory = join(stateDir, "marimo", "hosts-v1");+ await mkdir(directory, { recursive: true, mode: 0o700 });+ await write();+ expect(await collect({})).toEqual([record]);+ });++ it("finishes when the directory is absent or empty", async () => {+ expect(await collect()).toEqual([]);+ await rm(directory, { recursive: true });+ expect(await collect()).toEqual([]);+ expect(runPs).not.toHaveBeenCalled();+ });++ it("skips malformed and unsupported records, directories, and temporary files", async () => {+ await write();+ await write({ ...record, version: 2 }, "future.json");+ await write({ ...record, token: "" }, "empty-token.json");+ await write(+ { ...record, process: { pid: 1.5, started_at: "invalid" } },+ "invalid.json",+ );+ await write(null, "null.json");+ await write(record, "pending.tmp");+ await mkdir(join(directory, "directory.json"));+ await writeFile(join(directory, "broken.json"), "{", { mode: 0o600 });+ expect(await collect()).toEqual([record]);+ expect(runPs).toHaveBeenCalledTimes(1);+ });++ it.each([+ "https://127.0.0.1/api",+ "http://localhost/api",+ "http://example.com/api",+ "http://127.1/api",+ "http://2130706433/api",+ "http://127.0.0.2/api",+ "http://[0:0:0:0:0:0:0:1]/api",+ "http://127.0.0.1.evil/api",+ "http://user:[email protected]/api",+ "http://127.0.0.1:99999/api",+ "http://127.0.0.1/api?token=secret",+ "http://127.0.0.1/api#fragment",+ "http://127.0.0.1/\\evil",+ "http://127.0.0.1/api\n",+ "not a URL",+ ])("skips an invalid host URL: %s", async (url) => {+ await write({ ...record, url });+ expect(await collect()).toEqual([]);+ expect(runPs).not.toHaveBeenCalled();+ });++ it.each([0o640, 0o604, 0o620])(+ "skips files with nonprivate mode %o",+ async (mode) => {+ await chmod(await write(), mode);+ expect(await collect()).toEqual([]);+ },+ );++ it("skips nonprivate directories", async () => {+ await write();+ await chmod(directory, 0o755);+ expect(await collect()).toEqual([]);+ });++ it("skips directories belonging to another user", async () => {+ await write();+ vi.spyOn(process, "getuid").mockReturnValue(process.getuid!() + 1);+ expect(await collect()).toEqual([]);+ });++ it("does not follow symlinks to records or the discovery directory", async () => {+ await write(record, "target.tmp");+ await symlink(+ join(directory, "target.tmp"),+ join(directory, "link.json"),+ );+ expect(await collect()).toEqual([]);+ await rm(directory, { recursive: true });+ const target = join(stateDir, "target");+ await mkdir(target, { mode: 0o700 });+ await writeFile(join(target, "host.json"), JSON.stringify(record), {+ mode: 0o600,+ });+ await symlink(target, directory);+ expect(await collect()).toEqual([]);+ });++ it("skips vanished records", async () => {+ await write();+ await write(record, "other.json");+ const hosts = discoverHosts({ stateDir });+ expect((await hosts.next()).value).toEqual(record);+ await rm(join(directory, "host.json"));+ await rm(join(directory, "other.json"));+ expect((await hosts.next()).done).toBe(true);+ });++ it.each([+ "Wed Oct 21 12:34:54 2026 S",+ "Wed Oct 21 12:34:56 2026 Z",+ "",+ "malformed",+ ])("skips stale, zombie, or unverifiable processes: %s", async (output) => {+ await write();+ runPs.mockResolvedValue({ stdout: output, stderr: "" });+ expect(await collect()).toEqual([]);+ expect(await readdir(directory)).toEqual(["host.json"]);+ });++ it("skips processes that have exited", async () => {+ await write();+ runPs.mockRejectedValue(+ Object.assign(new Error("exited"), {+ code: 1,+ stdout: "",+ stderr: "",+ }),+ );+ expect(await collect()).toEqual([]);+ });++ it.each([+ { code: "ENOENT", stdout: "", stderr: "" },+ { code: 1, stdout: "", stderr: "ps: invalid option -- 'p'\n" },+ { code: 1, stdout: "Usage: ps [-o COL1,COL2=HEADER]\n", stderr: "" },+ { code: 2, stdout: "", stderr: "" },+ ])("reports process inspection failures: %j", async (details) => {+ await write();+ const error = Object.assign(new Error("ps failed"), details);+ runPs.mockRejectedValue(error);+ await expect(collect()).rejects.toBe(error);+ });++ it("verifies a real running process", async () => {+ const { execFile } =+ await vi.importActual<typeof import("node:child_process")>(+ "node:child_process",+ );+ runPs.mockImplementation(promisify(execFile));+ const live = {+ ...record,+ process: {+ pid: process.pid,+ started_at: new Date(+ Date.now() - process.uptime() * 1000,+ ).toISOString(),+ },+ };+ await write(live);+ expect(await collect()).toEqual([live]);+ });+ },+);diff --git a/packages/client/src/node.ts b/packages/client/src/node.tsnew file mode 100644index 0000000000..2664bb60b4--- /dev/null+++ b/packages/client/src/node.ts@@ -0,0 +1,3 @@+/* Copyright 2026 Marimo. All rights reserved. */++export { discoverHosts, type DiscoverHostsOptions } from "./node/discovery.ts";diff --git a/packages/client/src/node/discovery.ts b/packages/client/src/node/discovery.tsnew file mode 100644index 0000000000..73e0d5ca4d--- /dev/null+++ b/packages/client/src/node/discovery.ts@@ -0,0 +1,178 @@+/* Copyright 2026 Marimo. All rights reserved. */++import { execFile } from "node:child_process";+import { constants, type Stats } from "node:fs";+import { lstat, open, readdir } from "node:fs/promises";+import { homedir } from "node:os";+import { join } from "node:path";+import process from "node:process";+import { promisify } from "node:util";+import { HostRecord, type HostRecordOutput } from "../protocol.ts";++const exec = promisify(execFile);++export interface DiscoverHostsOptions {+ /** The marimo state directory containing `hosts-v1`. Defaults to+ * `$XDG_STATE_HOME/marimo`, or `~/.local/state/marimo` when unset. */+ stateDir?: string;+}++/**+ * Discover running hosts on this machine with a single scan of their records.+ *+ * Yields records with a private directory and file owned by the current user,+ * a loopback URL, and a matching live process. Invalid, inaccessible, and stale+ * records are skipped without removing them. A missing directory yields nothing.+ * No HTTP requests are made; a host can stop after it has been discovered.+ *+ * Supports macOS and Linux, using `/bin/ps` to verify process start times.+ * Linux requires procps; BusyBox ps is not supported.+ * Throws on unsupported platforms or if process inspection fails.+ */+export async function* discoverHosts(+ options: DiscoverHostsOptions = {},+): AsyncGenerator<HostRecordOutput> {+ if (process.platform !== "darwin" && process.platform !== "linux") {+ throw new Error("discoverHosts() supports macOS and Linux.");+ }+ const stateHome = process.env.XDG_STATE_HOME;+ const directory = join(+ options.stateDir ??+ join(+ stateHome?.trim() ? stateHome : join(homedir(), ".local", "state"),+ "marimo",+ ),+ "hosts-v1",+ );+ const uid = process.getuid!();+ let entries;+ let directoryStat: Stats;+ try {+ directoryStat = await lstat(directory);+ if (!directoryStat.isDirectory() || !isPrivate(directoryStat, uid)) return;+ entries = await readdir(directory, { withFileTypes: true });+ } catch (error) {+ if (isUnavailable(error)) return;+ throw error;+ }++ for (const entry of entries) {+ if (!entry.isFile() || !entry.name.endsWith(".json")) continue;+ const record = await readRecord(join(directory, entry.name), uid);+ if (!record) continue;+ // Do not trust records if the directory was replaced during the scan.+ try {+ const current = await lstat(directory);+ if (+ current.dev !== directoryStat.dev ||+ current.ino !== directoryStat.ino ||+ !isPrivate(current, uid)+ )+ return;+ } catch (error) {+ if (isUnavailable(error)) return;+ throw error;+ }+ if (await isRunning(record.process)) yield record;+ }+}++function isPrivate(stat: Stats, uid: number): boolean {+ return stat.uid === uid && (stat.mode & 0o077) === 0;+}++function isUnavailable(error: unknown): boolean {+ return (+ error instanceof Error &&+ "code" in error &&+ ["ENOENT", "ENOTDIR", "EACCES", "EPERM", "ELOOP"].includes(+ String(error.code),+ )+ );+}++async function readRecord(+ path: string,+ uid: number,+): Promise<HostRecordOutput | undefined> {+ try {+ // Check the opened file, and never follow a symlink substituted for it.+ // NONBLOCK also prevents a substituted FIFO from hanging discovery.+ const file = await open(+ path,+ constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,+ );+ try {+ const stat = await file.stat();+ if (!stat.isFile() || !isPrivate(stat, uid)) return;+ const result = HostRecord.safeParse(+ JSON.parse(await file.readFile("utf8")),+ );+ if (result.success && isLoopback(result.data.url)) return result.data;+ } finally {+ await file.close();+ }+ } catch (error) {+ if (error instanceof SyntaxError || isUnavailable(error)) return;+ throw error;+ }+}++function isLoopback(value: string): boolean {+ // Match the literal authority before URL parsing can normalize other IPv4+ // spellings (e.g. 127.1) or resolve backslashes into a different URL.+ if (!/^http:\/\/(127\.0\.0\.1|\[::1\])(?::[0-9]+)?(?:\/|$)/.test(value)) {+ return false;+ }+ try {+ const url = new URL(value);+ return (+ (url.hostname === "127.0.0.1" || url.hostname === "[::1]") &&+ !url.username &&+ !url.password &&+ !url.search &&+ !url.hash &&+ !/[\\\s?#]/.test(value)+ );+ } catch {+ return false;+ }+}++async function isRunning(+ identity: HostRecordOutput["process"],+): Promise<boolean> {+ let stdout: string;+ try {+ ({ stdout } = await exec(+ "/bin/ps",+ ["-p", String(identity.pid), "-o", "lstart=", "-o", "stat="],+ {+ env: { ...process.env, LC_ALL: "C", TZ: "UTC" },+ timeout: 1000,+ maxBuffer: 4096,+ },+ ));+ } catch (error) {+ // An empty result exits with status 1. Diagnostics indicate an inspection+ // failure (for example, unsupported ps options), not a dead process.+ if (+ error instanceof Error &&+ "code" in error &&+ error.code === 1 &&+ "stdout" in error &&+ error.stdout === "" &&+ "stderr" in error &&+ error.stderr === ""+ ) {+ return false;+ }+ throw error;+ }+ const match = /^(.*?)\s+(\S+)\s*$/.exec(stdout.trim());+ if (!match || /^[ZX]/.test(match[2])) return false;+ // ps reports whole seconds. Match the protocol's subsecond timestamp with+ // the same tolerance as the host conformance suite.+ const started = Date.parse(`${match[1]} UTC`);+ return Math.abs(started - Date.parse(identity.started_at)) < 1000;+}diff --git a/packages/client/tsconfig.build.json b/packages/client/tsconfig.build.jsonindex 0c0beed6fc..a1f1dae40b 100644--- a/packages/client/tsconfig.build.json+++ b/packages/client/tsconfig.build.json@@ -5,7 +5,7 @@"declaration": true,"rootDir": "src","outDir": "dist",- "types": []+ "types": ["node"]},"include": ["src"],"exclude": ["src/**/__tests__"]