Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: backend data source discovery (#10408)

Myles Scolnicka688c9180d31
diff --git a/frontend/src/__mocks__/requests.ts b/frontend/src/__mocks__/requests.ts
index 0d47f505c7..745ea0105c 100644
--- a/frontend/src/__mocks__/requests.ts
+++ b/frontend/src/__mocks__/requests.ts
@@ -41,6 +41,7 @@ export const MockRequestClient = {
previewSQLTableList: vi.fn().mockResolvedValue({ tables: [] }),
previewSQLSchemaList: vi.fn().mockResolvedValue({ schemas: [] }),
previewDataSourceConnection: vi.fn().mockResolvedValue({}),
+ discoverDataSources: vi.fn().mockResolvedValue({}),
validateSQL: vi.fn().mockResolvedValue({}),
openFile: vi.fn().mockResolvedValue({}),
getUsageStats: vi.fn().mockResolvedValue({}),
diff --git a/frontend/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx b/frontend/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx
new file mode 100644
index 0000000000..3d21d621a8
--- /dev/null
+++ b/frontend/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx
@@ -0,0 +1,113 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { describe, expect, it, vi } from "vitest";
+import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
+import { QuickAddDataSources } from "../quick-add-data-sources";
+
+const sources: DetectedDataSource[] = [
+ {
+ id: "postgres-libpq-environment",
+ integration: "postgres",
+ category: "database",
+ displayName: "PostgreSQL",
+ confidence: "high",
+ origins: [{ type: "environment", label: "Kernel environment" }],
+ configuration: [
+ {
+ field: "Host",
+ value: { kind: "environment-variable", name: "PGHOST" },
+ },
+ {
+ field: "Username",
+ value: { kind: "environment-variable", name: "PGUSER" },
+ },
+ {
+ field: "Database",
+ value: { kind: "environment-variable", name: "PGDATABASE" },
+ },
+ ],
+ code: "engine = create_engine()",
+ },
+ {
+ id: "pyiceberg-prod",
+ integration: "pyiceberg",
+ category: "catalog",
+ displayName: "PyIceberg (prod)",
+ confidence: "high",
+ origins: [
+ {
+ type: "configuration",
+ label: "Resolved PyIceberg configuration",
+ },
+ ],
+ configuration: [
+ {
+ field: "Catalog",
+ value: { kind: "safe-literal", value: "prod" },
+ },
+ {
+ field: "Type",
+ value: { kind: "safe-literal", value: "REST" },
+ },
+ ],
+ code: 'catalog = load_catalog("prod")',
+ },
+];
+
+describe("QuickAddDataSources", () => {
+ it("does not render an empty section", () => {
+ const { container } = render(
+ <QuickAddDataSources sources={[]} onAdd={vi.fn()} />,
+ );
+
+ expect(container).toBeEmptyDOMElement();
+ });
+
+ it("renders detected sources as clickable tags", () => {
+ const onAdd = vi.fn();
+
+ render(<QuickAddDataSources sources={sources} onAdd={onAdd} />);
+ fireEvent.click(
+ screen.getByRole("button", {
+ name: "Add PostgreSQL connection",
+ }),
+ );
+
+ expect(screen.getByText("Quick add")).toBeInTheDocument();
+ expect(onAdd).toHaveBeenCalledWith(sources[0]);
+ });
+
+ it("shows environment references on hover", async () => {
+ render(<QuickAddDataSources sources={sources} onAdd={vi.fn()} />);
+ const tag = screen.getByRole("button", {
+ name: "Add PostgreSQL connection",
+ });
+
+ fireEvent.pointerMove(tag);
+ fireEvent.mouseOver(tag);
+
+ await waitFor(() => {
+ expect(screen.getAllByText('os.environ["PGHOST"]')).not.toHaveLength(0);
+ });
+ expect(screen.getAllByText('os.environ["PGUSER"]')).not.toHaveLength(0);
+ expect(screen.getAllByText('os.environ["PGDATABASE"]')).not.toHaveLength(0);
+ });
+
+ it("shows safe configuration metadata on hover", async () => {
+ render(<QuickAddDataSources sources={sources} onAdd={vi.fn()} />);
+ const tag = screen.getByRole("button", {
+ name: "Add PyIceberg (prod) connection",
+ });
+
+ fireEvent.pointerMove(tag);
+ fireEvent.mouseOver(tag);
+
+ await waitFor(() => {
+ expect(
+ screen.getAllByText("Detected from Resolved PyIceberg configuration"),
+ ).not.toHaveLength(0);
+ });
+ expect(screen.getAllByText("REST")).not.toHaveLength(0);
+ });
+});
diff --git a/frontend/src/components/editor/connections/add-connection-dialog.tsx b/frontend/src/components/editor/connections/add-connection-dialog.tsx
index 15d2a19a04..dd90a6c48e 100644
--- a/frontend/src/components/editor/connections/add-connection-dialog.tsx
+++ b/frontend/src/components/editor/connections/add-connection-dialog.tsx
@@ -12,6 +12,7 @@ import {
import { ExternalLink } from "@/components/ui/links";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { AddDatabaseForm } from "./database/add-database-form";
+import { AutoDiscoveredDataSources } from "./quick-add-data-sources";
import { AddStorageForm } from "./storage/add-storage-form";
type ConnectionTab = "databases" | "storage";
@@ -86,6 +87,7 @@ export const AddConnectionDialogContent: React.FC<{
<span className="block">{codeSnippetHint}</span>
</DialogDescription>
</DialogHeader>
+ <AutoDiscoveredDataSources onSubmit={onClose} />
<Tabs
value={activeTab}
onValueChange={(v) => setActiveTab(v as ConnectionTab)}
diff --git a/frontend/src/components/editor/connections/quick-add-data-sources.tsx b/frontend/src/components/editor/connections/quick-add-data-sources.tsx
new file mode 100644
index 0000000000..e2024c2e7a
--- /dev/null
+++ b/frontend/src/components/editor/connections/quick-add-data-sources.tsx
@@ -0,0 +1,106 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { PlusIcon, SparklesIcon } from "lucide-react";
+import { Tooltip, TooltipProvider } from "@/components/ui/tooltip";
+import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
+import { useDataSourceDiscovery } from "@/hooks/useDataSourceDiscovery";
+import { useInsertCode } from "./components";
+
+export const QuickAddDataSources: React.FC<{
+ sources: DetectedDataSource[];
+ onAdd: (source: DetectedDataSource) => void;
+}> = ({ sources, onAdd }) => {
+ if (sources.length === 0) {
+ return null;
+ }
+
+ return (
+ <section
+ aria-labelledby="quick-add-data-sources-title"
+ className="rounded-md border bg-muted/30 px-3 py-2"
+ >
+ <div className="flex flex-wrap items-center gap-2">
+ <div className="mr-1 flex items-center gap-1.5">
+ <SparklesIcon className="h-3.5 w-3.5 text-muted-foreground" />
+ <h3 id="quick-add-data-sources-title" className="text-sm font-medium">
+ Quick add
+ </h3>
+ </div>
+ <TooltipProvider delayDuration={200}>
+ {sources.map((source) => (
+ <Tooltip
+ key={source.id}
+ side="bottom"
+ content={<DetectedDataSourceDetails source={source} />}
+ >
+ <button
+ type="button"
+ aria-label={`Add ${source.displayName} connection`}
+ className="inline-flex items-center gap-1 rounded-full border border-(--blue-8) bg-(--blue-2) px-2.5 py-1 text-xs font-semibold text-(--blue-11) transition-colors hover:bg-(--blue-3) focus:outline-hidden focus:ring-2 focus:ring-ring focus:ring-offset-2"
+ onClick={() => onAdd(source)}
+ >
+ <PlusIcon className="h-3 w-3" />
+ {source.displayName}
+ </button>
+ </Tooltip>
+ ))}
+ </TooltipProvider>
+ </div>
+ </section>
+ );
+};
+
+const DetectedDataSourceDetails: React.FC<{
+ source: DetectedDataSource;
+}> = ({ source }) => (
+ <div className="min-w-64 space-y-2 py-1">
+ <div>
+ <div className="font-medium">{source.displayName}</div>
+ <div className="text-xs text-muted-foreground">
+ Detected from{" "}
+ {source.origins.map((origin) => origin.label).join(" and ")}
+ </div>
+ </div>
+ <dl className="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1 text-xs">
+ {source.configuration.map((item) => (
+ <div
+ className="contents"
+ key={
+ item.value.kind === "environment-variable"
+ ? item.value.name
+ : `${item.field}:${item.value.value}`
+ }
+ >
+ <dt className="text-muted-foreground">{item.field}</dt>
+ <dd>
+ <code>
+ {item.value.kind === "environment-variable"
+ ? `os.environ["${item.value.name}"]`
+ : item.value.value}
+ </code>
+ </dd>
+ </div>
+ ))}
+ </dl>
+ <div className="text-xs text-muted-foreground">
+ Click to add a configured cell.
+ </div>
+ </div>
+);
+
+export const AutoDiscoveredDataSources: React.FC<{
+ onSubmit: () => void;
+}> = ({ onSubmit }) => {
+ const insertCode = useInsertCode();
+ const { data } = useDataSourceDiscovery();
+
+ return (
+ <QuickAddDataSources
+ sources={data ?? []}
+ onAdd={(source) => {
+ insertCode(source.code);
+ onSubmit();
+ }}
+ />
+ );
+};
diff --git a/frontend/src/core/datasets/data-source-discovery.ts b/frontend/src/core/datasets/data-source-discovery.ts
new file mode 100644
index 0000000000..f5931322a1
--- /dev/null
+++ b/frontend/src/core/datasets/data-source-discovery.ts
@@ -0,0 +1,5 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import type { components } from "@marimo-team/marimo-api";
+
+export type DetectedDataSource = components["schemas"]["DetectedDataSource"];
diff --git a/frontend/src/core/datasets/request-registry.ts b/frontend/src/core/datasets/request-registry.ts
index 938e3cb93f..50d40e50ba 100644
--- a/frontend/src/core/datasets/request-registry.ts
+++ b/frontend/src/core/datasets/request-registry.ts
@@ -1,5 +1,6 @@
/* Copyright 2026 Marimo. All rights reserved. */
import type {
+ DataSourceDiscoveryResult,
SQLSchemaListPreview,
SQLTableListPreview,
SQLTablePreview,
@@ -19,6 +20,16 @@ import type {
// The backend returns data tables, which could also exist in other engines, dbs, schemas
// Thus, we use the request ID pattern to match the response to the request
+export const DiscoverDataSources = new DeferredRequestRegistry<
+ {},
+ DataSourceDiscoveryResult
+>("data-source-discovery-result", async (requestId) => {
+ const client = getRequestClient();
+ await client.discoverDataSources({
+ requestId,
+ });
+});
+
export const PreviewSQLTable = new DeferredRequestRegistry<
Omit<PreviewSQLTableRequest, "requestId">,
SQLTablePreview
diff --git a/frontend/src/core/islands/bootstrap.ts b/frontend/src/core/islands/bootstrap.ts
index 6dc2e6aa01..81c2620c3d 100644
--- a/frontend/src/core/islands/bootstrap.ts
+++ b/frontend/src/core/islands/bootstrap.ts
@@ -194,6 +194,7 @@ function handleMessage(
case "sql-schema-list-preview":
case "datasets":
case "data-source-connections":
+ case "data-source-discovery-result":
case "validate-sql-result":
case "storage-namespaces":
case "storage-entries":
diff --git a/frontend/src/core/islands/bridge.ts b/frontend/src/core/islands/bridge.ts
index 2c363fe78d..82440167ae 100644
--- a/frontend/src/core/islands/bridge.ts
+++ b/frontend/src/core/islands/bridge.ts
@@ -341,6 +341,7 @@ export class IslandsPyodideBridge implements RunRequests, EditRequests {
previewSQLTableList = throwNotImplemented;
previewSQLSchemaList = throwNotImplemented;
previewDataSourceConnection = throwNotImplemented;
+ discoverDataSources = throwNotImplemented;
validateSQL = throwNotImplemented;
openFile = throwNotImplemented;
sendListFiles = throwNotImplemented;
diff --git a/frontend/src/core/kernel/messages.ts b/frontend/src/core/kernel/messages.ts
index 2e7a819dbc..653db1dff0 100644
--- a/frontend/src/core/kernel/messages.ts
+++ b/frontend/src/core/kernel/messages.ts
@@ -44,6 +44,8 @@ export type SQLTableListPreview =
export type SQLSchemaListPreview =
NotificationMessageData<"sql-schema-list-preview">;
export type ValidateSQLResult = NotificationMessageData<"validate-sql-result">;
+export type DataSourceDiscoveryResult =
+ NotificationMessageData<"data-source-discovery-result">;
export type SecretKeysResult = NotificationMessageData<"secret-keys-result">;
export type StartupLogs = NotificationMessageData<"startup-logs">;
export type CellMessage = NotificationMessageData<"cell-op">;
diff --git a/frontend/src/core/network/__tests__/requests-lazy.test.ts b/frontend/src/core/network/__tests__/requests-lazy.test.ts
index bd12d68a3b..5d046de00b 100644
--- a/frontend/src/core/network/__tests__/requests-lazy.test.ts
+++ b/frontend/src/core/network/__tests__/requests-lazy.test.ts
@@ -114,6 +114,21 @@ describe("createLazyRequests", () => {
expect(mockInit).toHaveBeenCalledTimes(1);
});
+ it("starts the kernel for datasource discovery", async () => {
+ mockDelegate.discoverDataSources = vi.fn().mockResolvedValue(null);
+ const lazyRequests = createLazyRequests(
+ mockDelegate,
+ mockGetRuntimeManager,
+ );
+
+ await lazyRequests.discoverDataSources({
+ requestId: requestId("discovery"),
+ });
+
+ expect(mockInit).toHaveBeenCalledOnce();
+ expect(mockDelegate.discoverDataSources).toHaveBeenCalledOnce();
+ });
+
it("should only call init once across multiple requests", async () => {
const lazyRequests = createLazyRequests(
mockDelegate,
diff --git a/frontend/src/core/network/__tests__/requests-network.test.ts b/frontend/src/core/network/__tests__/requests-network.test.ts
index 0c0350c77c..5f5f363a8e 100644
--- a/frontend/src/core/network/__tests__/requests-network.test.ts
+++ b/frontend/src/core/network/__tests__/requests-network.test.ts
@@ -130,6 +130,20 @@ describe("createNetworkRequests", () => {
expect(mockClient.GET).toHaveBeenCalledWith("/api/export/availability");
});
+ it("discoverDataSources should POST to the discovery endpoint", async () => {
+ const requests = createNetworkRequests();
+ const request = { requestId: "discovery-request" } as any;
+ await requests.discoverDataSources(request);
+
+ expect(mockClient.POST).toHaveBeenCalledWith(
+ "/api/datasources/discover",
+ expect.objectContaining({
+ body: request,
+ params: expect.anything(),
+ }),
+ );
+ });
+
it("getPackageList should not require a kernel connection", async () => {
const { waitForConnectionOpen, waitForConnectionOpenIfNotebook } =
await import("../connection");
diff --git a/frontend/src/core/network/requests-lazy.ts b/frontend/src/core/network/requests-lazy.ts
index 521b093d61..e086f5a047 100644
--- a/frontend/src/core/network/requests-lazy.ts
+++ b/frontend/src/core/network/requests-lazy.ts
@@ -51,6 +51,7 @@ const ACTIONS: Record<keyof AllRequests, Action> = {
sendRunScratchpad: "startConnection",
saveAppConfig: "startConnection",
saveCellConfig: "startConnection",
+ discoverDataSources: "startConnection",
// Export operations start a connection
exportAsHTML: "startConnection",
diff --git a/frontend/src/core/network/requests-network.ts b/frontend/src/core/network/requests-network.ts
index 578fc20a16..ae1e07e92e 100644
--- a/frontend/src/core/network/requests-network.ts
+++ b/frontend/src/core/network/requests-network.ts
@@ -267,6 +267,14 @@ export function createNetworkRequests(): EditRequests & RunRequests {
})
.then(handleResponseReturnNull);
},
+ discoverDataSources: (request) => {
+ return getClient()
+ .POST("/api/datasources/discover", {
+ body: request,
+ params: getParams(),
+ })
+ .then(handleResponseReturnNull);
+ },
validateSQL: (request) => {
return getClient()
.POST("/api/sql/validate", {
diff --git a/frontend/src/core/network/requests-static.ts b/frontend/src/core/network/requests-static.ts
index 7e3e0c1280..045e71b2bb 100644
--- a/frontend/src/core/network/requests-static.ts
+++ b/frontend/src/core/network/requests-static.ts
@@ -62,6 +62,7 @@ export function createStaticRequests(): EditRequests & RunRequests {
previewSQLTableList: throwNotInEditMode,
previewSQLSchemaList: throwNotInEditMode,
previewDataSourceConnection: throwNotInEditMode,
+ discoverDataSources: throwNotInEditMode,
validateSQL: throwNotInEditMode,
openFile: throwNotInEditMode,
getUsageStats: throwNotInEditMode,
diff --git a/frontend/src/core/network/requests-toasting.tsx b/frontend/src/core/network/requests-toasting.tsx
index f5ccf83df7..2dec419086 100644
--- a/frontend/src/core/network/requests-toasting.tsx
+++ b/frontend/src/core/network/requests-toasting.tsx
@@ -43,6 +43,7 @@ export function createErrorToastingRequests(
previewSQLTableList: "Failed to fetch SQL table list",
previewSQLSchemaList: "Failed to fetch SQL schema list",
previewDataSourceConnection: "Failed to preview data source connection",
+ discoverDataSources: "Failed to discover data sources",
validateSQL: "Failed to validate SQL",
openFile: "Failed to open file",
getUsageStats: "", // No toast
diff --git a/frontend/src/core/network/types.ts b/frontend/src/core/network/types.ts
index a95404b338..17e57ffcc4 100644
--- a/frontend/src/core/network/types.ts
+++ b/frontend/src/core/network/types.ts
@@ -75,6 +75,7 @@ export type ListSQLTablesRequest = schemas["ListSQLTablesRequest"];
export type ListSQLSchemasRequest = schemas["ListSQLSchemasRequest"];
export type ListDataSourceConnectionRequest =
schemas["ListDataSourceConnectionRequest"];
+export type DiscoverDataSourcesRequest = schemas["DiscoverDataSourcesRequest"];
export type ValidateSQLRequest = schemas["ValidateSQLRequest"];
export type DebugCellRequest = schemas["DebugCellRequest"];
export type SetBreakpointsRequest = schemas["SetBreakpointsRequest"];
@@ -181,6 +182,7 @@ export interface EditRequests {
previewDataSourceConnection: (
request: ListDataSourceConnectionRequest,
) => Promise<null>;
+ discoverDataSources: (request: DiscoverDataSourcesRequest) => Promise<null>;
validateSQL: (request: ValidateSQLRequest) => Promise<null>;
openFile: (request: { path: string; lineNumber?: number }) => Promise<null>;
getUsageStats: () => Promise<UsageResponse>;
diff --git a/frontend/src/core/wasm/bridge.ts b/frontend/src/core/wasm/bridge.ts
index 596749e7fa..97d9d76220 100644
--- a/frontend/src/core/wasm/bridge.ts
+++ b/frontend/src/core/wasm/bridge.ts
@@ -630,6 +630,16 @@ export class PyodideBridge implements RunRequests, EditRequests {
return null;
};
+ discoverDataSources: EditRequests["discoverDataSources"] = async (
+ request,
+ ) => {
+ await this.putControlRequest({
+ type: "discover-data-sources",
+ ...request,
+ });
+ return null;
+ };
+
getUsageStats = throwNotImplemented;
getEnvironmentInfo: EditRequests["getEnvironmentInfo"] = async () => {
const response = await this.rpc.proxy.request.bridge({
diff --git a/frontend/src/core/websocket/useMarimoKernelConnection.tsx b/frontend/src/core/websocket/useMarimoKernelConnection.tsx
index 6bde5e3206..261d998abb 100644
--- a/frontend/src/core/websocket/useMarimoKernelConnection.tsx
+++ b/frontend/src/core/websocket/useMarimoKernelConnection.tsx
@@ -42,6 +42,7 @@ import { connectionTransportTypeAtom, useSetAppConfig } from "../config/config";
import { useDataSourceActions } from "../datasets/data-source-connections";
import type { ConnectionName } from "../datasets/engines";
import {
+ DiscoverDataSources,
PreviewSQLSchemaList,
PreviewSQLTable,
PreviewSQLTableList,
@@ -394,6 +395,9 @@ export function useMarimoKernelConnection(opts: {
case "validate-sql-result":
ValidateSQL.resolve(msg.data.request_id as RequestId, msg.data);
return;
+ case "data-source-discovery-result":
+ DiscoverDataSources.resolve(msg.data.request_id as RequestId, msg.data);
+ return;
case "secret-keys-result":
SECRETS_REGISTRY.resolve(msg.data.request_id, msg.data);
return;
diff --git a/frontend/src/hooks/__tests__/useDataSourceDiscovery.test.ts b/frontend/src/hooks/__tests__/useDataSourceDiscovery.test.ts
new file mode 100644
index 0000000000..42a0bfb8b3
--- /dev/null
+++ b/frontend/src/hooks/__tests__/useDataSourceDiscovery.test.ts
@@ -0,0 +1,80 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { DiscoverDataSources } from "@/core/datasets/request-registry";
+import { loadDataSourceDiscovery } from "../useDataSourceDiscovery";
+
+describe("loadDataSourceDiscovery", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("returns the kernel's secret-free discovery model", async () => {
+ const sources = [
+ {
+ id: "pyiceberg-prod",
+ integration: "pyiceberg",
+ category: "catalog" as const,
+ displayName: "PyIceberg (prod)",
+ confidence: "high" as const,
+ origins: [
+ {
+ type: "configuration" as const,
+ label: "Resolved PyIceberg configuration",
+ },
+ ],
+ configuration: [
+ {
+ field: "Catalog",
+ value: {
+ kind: "safe-literal" as const,
+ value: "prod",
+ },
+ },
+ ],
+ code: 'catalog = load_catalog("prod")',
+ },
+ ];
+ const request = vi.spyOn(DiscoverDataSources, "request").mockResolvedValue({
+ request_id: "request-id",
+ sources,
+ });
+
+ const detected = await loadDataSourceDiscovery();
+
+ expect({ detected, requests: request.mock.calls }).toMatchInlineSnapshot(`
+ {
+ "detected": [
+ {
+ "category": "catalog",
+ "code": "catalog = load_catalog("prod")",
+ "confidence": "high",
+ "configuration": [
+ {
+ "field": "Catalog",
+ "value": {
+ "kind": "safe-literal",
+ "value": "prod",
+ },
+ },
+ ],
+ "displayName": "PyIceberg (prod)",
+ "id": "pyiceberg-prod",
+ "integration": "pyiceberg",
+ "origins": [
+ {
+ "label": "Resolved PyIceberg configuration",
+ "type": "configuration",
+ },
+ ],
+ },
+ ],
+ "requests": [
+ [
+ {},
+ ],
+ ],
+ }
+ `);
+ });
+});
diff --git a/frontend/src/hooks/useDataSourceDiscovery.ts b/frontend/src/hooks/useDataSourceDiscovery.ts
new file mode 100644
index 0000000000..fcf41f2d62
--- /dev/null
+++ b/frontend/src/hooks/useDataSourceDiscovery.ts
@@ -0,0 +1,18 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
+import { DiscoverDataSources } from "@/core/datasets/request-registry";
+import { useAsyncData } from "./useAsyncData";
+
+export async function loadDataSourceDiscovery(): Promise<DetectedDataSource[]> {
+ const result = await DiscoverDataSources.request({});
+ return result.sources;
+}
+
+/**
+ * Reusable UI-facing hook for kernel-managed datasource discovery.
+ * Consumers decide how to render, filter, or act on the detected model.
+ */
+export function useDataSourceDiscovery() {
+ return useAsyncData(loadDataSourceDiscovery, []);
+}
diff --git a/marimo/_cli/cli.py b/marimo/_cli/cli.py
index b3355bc3dd..0b7fff5b33 100644
--- a/marimo/_cli/cli.py
+++ b/marimo/_cli/cli.py
@@ -251,7 +251,7 @@ edit_help_msg = "\n".join(
"\b",
"Create or edit notebooks.",
"\b",
- "If NAME is a url, the notebook will be downloaded to a temporary file."
+ "If NAME is a url, the notebook will be downloaded to a temporary file.",
"",
_key_value_bullets(
[
diff --git a/marimo/_cli/development/commands.py b/marimo/_cli/development/commands.py
index 7f0de39184..e12dcabcea 100644
--- a/marimo/_cli/development/commands.py
+++ b/marimo/_cli/development/commands.py
@@ -401,6 +401,7 @@ def _generate_server_api_schema() -> dict[str, Any]:
commands.CodeCompletionCommand,
commands.DebugCellCommand,
commands.DeleteCellCommand,
+ commands.DiscoverDataSourcesCommand,
commands.ExecuteCellsCommand,
commands.ExecuteScratchpadCommand,
commands.ExecuteStaleCellsCommand,
@@ -428,6 +429,7 @@ def _generate_server_api_schema() -> dict[str, Any]:
models.CopyNotebookRequest,
models.DebugCellRequest,
models.DeleteCellRequest,
+ models.DiscoverDataSourcesRequest,
models.ExecuteScratchpadRequest,
models.FormatCellsRequest,
models.FormatResponse,
diff --git a/marimo/_data/data_source_discovery/__init__.py b/marimo/_data/data_source_discovery/__init__.py
new file mode 100644
index 0000000000..c2af63a4ac
--- /dev/null
+++ b/marimo/_data/data_source_discovery/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2026 Marimo. All rights reserved.
+from marimo._data.data_source_discovery.discover import discover_data_sources
+from marimo._data.data_source_discovery.models import (
+ DetectedDataSource,
+ DetectedDataSourceConfiguration,
+ DetectedDataSourceOrigin,
+ EnvironmentVariableDiscoveryValue,
+ SafeLiteralDiscoveryValue,
+)
+
+__all__ = [
+ "DetectedDataSource",
+ "DetectedDataSourceConfiguration",
+ "DetectedDataSourceOrigin",
+ "EnvironmentVariableDiscoveryValue",
+ "SafeLiteralDiscoveryValue",
+ "discover_data_sources",
+]
diff --git a/marimo/_data/data_source_discovery/discover.py b/marimo/_data/data_source_discovery/discover.py
new file mode 100644
index 0000000000..1149e9b4d5
--- /dev/null
+++ b/marimo/_data/data_source_discovery/discover.py
@@ -0,0 +1,49 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from types import MappingProxyType
+from typing import TYPE_CHECKING
+
+from marimo import _loggers
+from marimo._data.data_source_discovery.models import DetectedDataSource
+from marimo._data.data_source_discovery.plugins import (
+ DEFAULT_DISCOVERY_PLUGINS,
+)
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping, Sequence
+
+LOGGER = _loggers.marimo_logger()
+
+
+def discover_data_sources(
+ environment: Mapping[str, str],
+ plugins: Sequence[DiscoveryPlugin] = DEFAULT_DISCOVERY_PLUGINS,
+) -> list[DetectedDataSource]:
+ """Run isolated detectors without exposing secret values to the frontend."""
+ # Give every plugin the same immutable snapshot. A detector cannot mutate
+ # the live process environment or affect a detector that runs after it.
+ context = DiscoveryContext(environment=MappingProxyType(dict(environment)))
+ sources: list[DetectedDataSource] = []
+
+ for plugin in plugins:
+ try:
+ sources.extend(plugin.discover(context))
+ except Exception as exc:
+ LOGGER.warning(
+ "Datasource discovery plugin %s failed (%s)",
+ plugin.id,
+ type(exc).__name__,
+ )
+
+ seen: set[str] = set()
+ unique_sources: list[DetectedDataSource] = []
+ for source in sources:
+ if source.id not in seen:
+ seen.add(source.id)
+ unique_sources.append(source)
+ return unique_sources
diff --git a/marimo/_data/data_source_discovery/helpers.py b/marimo/_data/data_source_discovery/helpers.py
new file mode 100644
index 0000000000..01ff5a1547
--- /dev/null
+++ b/marimo/_data/data_source_discovery/helpers.py
@@ -0,0 +1,59 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from marimo._data.data_source_discovery.models import (
+ DetectedDataSourceConfiguration,
+ DetectedDataSourceOrigin,
+ EnvironmentVariableDiscoveryValue,
+ SafeLiteralDiscoveryValue,
+)
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping
+
+ENVIRONMENT_ORIGIN = DetectedDataSourceOrigin(
+ type="environment",
+ label="Kernel environment",
+)
+RESOLVED_CONFIGURATION_ORIGIN = DetectedDataSourceOrigin(
+ type="configuration",
+ label="Resolved PyIceberg configuration",
+)
+
+
+def has_value(environment: Mapping[str, str], name: str) -> bool:
+ """Return whether an environment variable has a non-empty value."""
+ return bool(environment.get(name))
+
+
+def has_all(environment: Mapping[str, str], names: tuple[str, ...]) -> bool:
+ return all(has_value(environment, name) for name in names)
+
+
+def is_valid_port(value: str | None) -> bool:
+ """Return whether a value is a valid integer network port."""
+ if value is None:
+ return False
+ try:
+ port = int(value)
+ except ValueError:
+ return False
+ return 1 <= port <= 65535
+
+
+def environment_variable(
+ field: str, name: str
+) -> DetectedDataSourceConfiguration:
+ return DetectedDataSourceConfiguration(
+ field=field,
+ value=EnvironmentVariableDiscoveryValue(name=name),
+ )
+
+
+def safe_literal(field: str, value: str) -> DetectedDataSourceConfiguration:
+ return DetectedDataSourceConfiguration(
+ field=field,
+ value=SafeLiteralDiscoveryValue(value=value),
+ )
diff --git a/marimo/_data/data_source_discovery/models.py b/marimo/_data/data_source_discovery/models.py
new file mode 100644
index 0000000000..8a1ba7129e
--- /dev/null
+++ b/marimo/_data/data_source_discovery/models.py
@@ -0,0 +1,64 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from typing import Literal
+
+import msgspec
+
+DataSourceCategory = Literal["database", "catalog", "object-storage"]
+DataSourceConfidence = Literal["high", "medium"]
+DataSourceDiscoveryOriginKind = Literal["environment", "configuration"]
+
+
+class EnvironmentVariableDiscoveryValue(
+ msgspec.Struct,
+ frozen=True,
+ tag="environment-variable",
+ tag_field="kind",
+ rename="camel",
+):
+ """A reference to an environment variable, never its value."""
+
+ name: str
+
+
+class SafeLiteralDiscoveryValue(
+ msgspec.Struct,
+ frozen=True,
+ tag="safe-literal",
+ tag_field="kind",
+ rename="camel",
+):
+ """Non-sensitive metadata that is safe to send to the frontend."""
+
+ value: str
+
+
+DataSourceDiscoveryValue = (
+ EnvironmentVariableDiscoveryValue | SafeLiteralDiscoveryValue
+)
+
+
+class DetectedDataSourceConfiguration(
+ msgspec.Struct, frozen=True, rename="camel"
+):
+ field: str
+ value: DataSourceDiscoveryValue
+
+
+class DetectedDataSourceOrigin(msgspec.Struct, frozen=True, rename="camel"):
+ type: DataSourceDiscoveryOriginKind
+ label: str
+
+
+class DetectedDataSource(msgspec.Struct, frozen=True, rename="camel"):
+ """A secret-free datasource suggestion produced by the kernel."""
+
+ id: str
+ integration: str
+ category: DataSourceCategory
+ display_name: str
+ confidence: DataSourceConfidence
+ origins: tuple[DetectedDataSourceOrigin, ...]
+ configuration: tuple[DetectedDataSourceConfiguration, ...]
+ code: str
diff --git a/marimo/_data/data_source_discovery/plugins/__init__.py b/marimo/_data/data_source_discovery/plugins/__init__.py
new file mode 100644
index 0000000000..3e73914087
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2026 Marimo. All rights reserved.
+from marimo._data.data_source_discovery.plugins.aws import AWS_PLUGIN
+from marimo._data.data_source_discovery.plugins.mysql import MYSQL_PLUGIN
+from marimo._data.data_source_discovery.plugins.postgres import POSTGRES_PLUGIN
+from marimo._data.data_source_discovery.plugins.pyiceberg import (
+ PYICEBERG_PLUGIN,
+)
+from marimo._data.data_source_discovery.plugins.pyspark import PYSPARK_PLUGIN
+from marimo._data.data_source_discovery.plugins.trino import TRINO_PLUGIN
+from marimo._data.data_source_discovery.types import DiscoveryPlugin
+
+DEFAULT_DISCOVERY_PLUGINS: tuple[DiscoveryPlugin, ...] = (
+ POSTGRES_PLUGIN,
+ MYSQL_PLUGIN,
+ AWS_PLUGIN,
+ TRINO_PLUGIN,
+ PYSPARK_PLUGIN,
+ PYICEBERG_PLUGIN,
+)
+
+__all__ = ["DEFAULT_DISCOVERY_PLUGINS"]
diff --git a/marimo/_data/data_source_discovery/plugins/aws.py b/marimo/_data/data_source_discovery/plugins/aws.py
new file mode 100644
index 0000000000..e5ab7867f5
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/aws.py
@@ -0,0 +1,74 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+ environment_variable,
+ has_all,
+ has_value,
+)
+from marimo._data.data_source_discovery.models import (
+ DetectedDataSource,
+ DetectedDataSourceConfiguration,
+)
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+ACCESS_KEYS = ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
+OPTIONAL_VARIABLES = (
+ ("Session token", "AWS_SESSION_TOKEN"),
+ ("Region", "AWS_DEFAULT_REGION"),
+ ("Region", "AWS_REGION"),
+ ("Profile", "AWS_PROFILE"),
+ ("Endpoint", "AWS_ENDPOINT_URL"),
+ ("S3 endpoint", "AWS_ENDPOINT_URL_S3"),
+)
+
+
+def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
+ environment = context.environment
+ has_access_keys = has_all(environment, ACCESS_KEYS)
+ has_profile = has_value(environment, "AWS_PROFILE")
+ if not has_access_keys and not has_profile:
+ return []
+
+ configuration: list[DetectedDataSourceConfiguration] = []
+ if has_access_keys:
+ configuration.extend(
+ (
+ environment_variable("Access key ID", "AWS_ACCESS_KEY_ID"),
+ environment_variable(
+ "Secret access key", "AWS_SECRET_ACCESS_KEY"
+ ),
+ )
+ )
+ for field, name in OPTIONAL_VARIABLES:
+ if has_value(environment, name):
+ configuration.append(environment_variable(field, name))
+
+ has_custom_endpoint = has_value(
+ environment, "AWS_ENDPOINT_URL"
+ ) or has_value(environment, "AWS_ENDPOINT_URL_S3")
+ return [
+ DetectedDataSource(
+ id="aws-s3-environment",
+ integration="aws",
+ category="object-storage",
+ display_name=(
+ "S3-compatible storage" if has_custom_endpoint else "Amazon S3"
+ ),
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=tuple(configuration),
+ code="""\
+import s3fs
+
+# Uses the standard AWS credential and endpoint environment variables.
+fs = s3fs.S3FileSystem()""",
+ )
+ ]
+
+
+AWS_PLUGIN = DiscoveryPlugin(id="aws", discover=discover)
diff --git a/marimo/_data/data_source_discovery/plugins/mysql.py b/marimo/_data/data_source_discovery/plugins/mysql.py
new file mode 100644
index 0000000000..7d4cca6f2f
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/mysql.py
@@ -0,0 +1,73 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+ environment_variable,
+ has_all,
+ has_value,
+ is_valid_port,
+)
+from marimo._data.data_source_discovery.models import DetectedDataSource
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+REQUIRED = ("MYSQL_HOST", "MYSQL_USER", "MYSQL_DATABASE")
+PASSWORD_VARIABLES = ("MYSQL_PASSWORD", "MYSQL_PWD")
+
+
+def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
+ environment = context.environment
+ password_variable = next(
+ (name for name in PASSWORD_VARIABLES if has_value(environment, name)),
+ None,
+ )
+ if not has_all(environment, REQUIRED) or password_variable is None:
+ return []
+
+ has_valid_port = is_valid_port(environment.get("MYSQL_TCP_PORT"))
+ configuration = [
+ environment_variable("Host", "MYSQL_HOST"),
+ environment_variable("Username", "MYSQL_USER"),
+ environment_variable("Password", password_variable),
+ environment_variable("Database", "MYSQL_DATABASE"),
+ ]
+ if has_valid_port:
+ configuration.append(environment_variable("Port", "MYSQL_TCP_PORT"))
+
+ port = (
+ ' port=int(os.environ["MYSQL_TCP_PORT"]),'
+ if has_valid_port
+ else " port=3306,"
+ )
+ code = f"""\
+import os
+import sqlalchemy
+
+DATABASE_URL = sqlalchemy.URL.create(
+ "mysql+pymysql",
+ username=os.environ["MYSQL_USER"],
+ password=os.environ["{password_variable}"],
+ host=os.environ["MYSQL_HOST"],
+{port}
+ database=os.environ["MYSQL_DATABASE"],
+)
+engine = sqlalchemy.create_engine(DATABASE_URL)"""
+
+ return [
+ DetectedDataSource(
+ id="mysql-environment",
+ integration="mysql",
+ category="database",
+ display_name="MySQL",
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=tuple(configuration),
+ code=code,
+ )
+ ]
+
+
+MYSQL_PLUGIN = DiscoveryPlugin(id="mysql", discover=discover)
diff --git a/marimo/_data/data_source_discovery/plugins/postgres.py b/marimo/_data/data_source_discovery/plugins/postgres.py
new file mode 100644
index 0000000000..44305bbf21
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/postgres.py
@@ -0,0 +1,77 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+ environment_variable,
+ has_all,
+ has_value,
+ is_valid_port,
+)
+from marimo._data.data_source_discovery.models import DetectedDataSource
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+REQUIRED = ("PGHOST", "PGUSER", "PGDATABASE")
+
+
+def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
+ environment = context.environment
+ if not has_all(environment, REQUIRED):
+ return []
+
+ has_valid_port = is_valid_port(environment.get("PGPORT"))
+ configuration = [
+ environment_variable("Host", "PGHOST"),
+ environment_variable("Username", "PGUSER"),
+ environment_variable("Database", "PGDATABASE"),
+ ]
+ if has_valid_port:
+ configuration.append(environment_variable("Port", "PGPORT"))
+ if has_value(environment, "PGPASSWORD"):
+ configuration.append(environment_variable("Password", "PGPASSWORD"))
+
+ port = (
+ ' port=int(os.environ["PGPORT"]),'
+ if has_valid_port
+ else " port=5432,"
+ )
+ password = (
+ [' password=os.environ["PGPASSWORD"],']
+ if has_value(environment, "PGPASSWORD")
+ else []
+ )
+ code = "\n".join(
+ [
+ "import os",
+ "import sqlalchemy",
+ "",
+ "DATABASE_URL = sqlalchemy.URL.create(",
+ ' "postgresql",',
+ ' username=os.environ["PGUSER"],',
+ *password,
+ ' host=os.environ["PGHOST"],',
+ port,
+ ' database=os.environ["PGDATABASE"],',
+ ")",
+ "engine = sqlalchemy.create_engine(DATABASE_URL)",
+ ]
+ )
+
+ return [
+ DetectedDataSource(
+ id="postgres-libpq-environment",
+ integration="postgres",
+ category="database",
+ display_name="PostgreSQL",
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=tuple(configuration),
+ code=code,
+ )
+ ]
+
+
+POSTGRES_PLUGIN = DiscoveryPlugin(id="postgres", discover=discover)
diff --git a/marimo/_data/data_source_discovery/plugins/pyiceberg.py b/marimo/_data/data_source_discovery/plugins/pyiceberg.py
new file mode 100644
index 0000000000..4f8191094d
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/pyiceberg.py
@@ -0,0 +1,353 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Callable, Mapping, Sequence
+from pathlib import Path
+from typing import cast
+
+import yaml
+
+from marimo import _loggers
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+ RESOLVED_CONFIGURATION_ORIGIN,
+ environment_variable,
+ safe_literal,
+)
+from marimo._data.data_source_discovery.models import (
+ DetectedDataSource,
+ DetectedDataSourceConfiguration,
+ DetectedDataSourceOrigin,
+)
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+from marimo._dependencies.dependencies import DependencyManager
+
+CATALOG_PREFIX = "PYICEBERG_CATALOG__"
+CONFIDENCE_PROPERTIES = frozenset({"py-catalog-impl", "type", "uri"})
+CONFIG_FILENAME = ".pyiceberg.yaml"
+CATALOG_TYPE_LABELS = {
+ "rest": "REST",
+ "hive": "Hive",
+ "glue": "Glue",
+ "dynamodb": "DynamoDB",
+ "sql": "SQL",
+ "in-memory": "In-memory",
+ "bigquery": "BigQuery",
+}
+
+CatalogConfiguration = Mapping[str, object]
+CatalogConfigurations = Mapping[str, CatalogConfiguration]
+CatalogConfigurationLoader = Callable[[], CatalogConfigurations]
+LOGGER = _loggers.marimo_logger()
+
+
+def discover(
+ context: DiscoveryContext,
+ load_catalogs: CatalogConfigurationLoader | None = None,
+) -> list[DetectedDataSource]:
+ environment_sources = _detect_environment_catalogs(context.environment)
+ try:
+ catalogs = (
+ load_catalogs()
+ if load_catalogs is not None
+ else _load_resolved_catalogs(environment=context.environment)
+ )
+ resolved_sources = _detect_resolved_catalogs(catalogs)
+ except Exception as exc:
+ LOGGER.warning(
+ "PyIceberg resolved configuration discovery failed (%s)",
+ type(exc).__name__,
+ )
+ resolved_sources = []
+ return _merge_catalogs([*environment_sources, *resolved_sources])
+
+
+def _detect_environment_catalogs(
+ environment: Mapping[str, str],
+) -> list[DetectedDataSource]:
+ catalogs: dict[str, list[tuple[str, str]]] = {}
+
+ for variable_name, value in environment.items():
+ if not value:
+ continue
+ if not variable_name.startswith(CATALOG_PREFIX):
+ continue
+
+ remainder = variable_name[len(CATALOG_PREFIX) :]
+ catalog_name, separator, property_name = remainder.partition("__")
+ if not separator:
+ continue
+
+ catalog_name = _normalize_key(catalog_name)
+ property_name = _normalize_key(property_name)
+ if not catalog_name or not property_name:
+ continue
+
+ catalogs.setdefault(catalog_name, []).append(
+ (property_name, variable_name)
+ )
+
+ sources: list[DetectedDataSource] = []
+ for catalog_name, properties in sorted(catalogs.items()):
+ if not any(
+ property_name in CONFIDENCE_PROPERTIES
+ for property_name, _ in properties
+ ):
+ continue
+
+ configuration = [safe_literal("Catalog", catalog_name)]
+ configuration.extend(
+ environment_variable(property_name, variable_name)
+ for property_name, variable_name in sorted(
+ properties, key=lambda item: item[1]
+ )
+ )
+ sources.append(
+ _iceberg_source(
+ catalog_name=catalog_name,
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=configuration,
+ )
+ )
+ return sources
+
+
+def _detect_resolved_catalogs(
+ catalogs: CatalogConfigurations,
+) -> list[DetectedDataSource]:
+ sources: list[DetectedDataSource] = []
+ for catalog_name, configuration in sorted(catalogs.items()):
+ if not catalog_name:
+ continue
+ catalog_type = _infer_catalog_type(configuration)
+ if catalog_type is None:
+ continue
+ sources.append(
+ _iceberg_source(
+ catalog_name=catalog_name,
+ origins=(RESOLVED_CONFIGURATION_ORIGIN,),
+ configuration=[
+ safe_literal("Catalog", catalog_name),
+ safe_literal("Type", catalog_type),
+ ],
+ )
+ )
+ return sources
+
+
+def _load_resolved_catalogs(
+ *,
+ environment: Mapping[str, str] | None = None,
+ home_directory: Path | None = None,
+ current_directory: Path | None = None,
+) -> CatalogConfigurations:
+ """Resolve catalogs with the same PyIceberg configuration used at runtime."""
+ # PyIceberg's `Config` reads the live process environment directly. Only
+ # use it when the caller has not supplied an environment snapshot.
+ if environment is None and DependencyManager.pyiceberg.has(quiet=True):
+ from pyiceberg.utils.config import Config
+
+ config = Config()
+ return {
+ name: cast(
+ CatalogConfiguration, config.get_catalog_config(name) or {}
+ )
+ for name in config.get_known_catalogs()
+ }
+
+ catalogs = _load_catalogs_from_configuration_file(
+ environment=os.environ if environment is None else environment,
+ home_directory=Path.home()
+ if home_directory is None
+ else home_directory,
+ current_directory=Path.cwd()
+ if current_directory is None
+ else current_directory,
+ )
+ return _apply_environment_catalog_overrides(
+ catalogs,
+ os.environ if environment is None else environment,
+ )
+
+
+def _load_catalogs_from_configuration_file(
+ *,
+ environment: Mapping[str, str],
+ home_directory: Path,
+ current_directory: Path,
+) -> CatalogConfigurations:
+ """Load the first `.pyiceberg.yaml` using PyIceberg's search order."""
+ search_directories = [
+ Path(configured_home)
+ for configured_home in [environment.get("PYICEBERG_HOME")]
+ if configured_home
+ ]
+ search_directories.extend((home_directory, current_directory))
+
+ for directory in search_directories:
+ path = directory / CONFIG_FILENAME
+ if not path.is_file():
+ continue
+
+ config = yaml.safe_load(path.read_text(encoding="utf-8"))
+ if config is None:
+ continue
+ if not config:
+ continue
+ return _catalogs_from_file_config(_lowercase_keys(config))
+
+ return {}
+
+
+def _apply_environment_catalog_overrides(
+ catalogs: CatalogConfigurations,
+ environment: Mapping[str, str],
+) -> CatalogConfigurations:
+ """Apply environment metadata to catalogs resolved from a config file."""
+ merged = {
+ name: dict(configuration) for name, configuration in catalogs.items()
+ }
+
+ for variable_name, value in environment.items():
+ if not variable_name.startswith(CATALOG_PREFIX):
+ continue
+
+ remainder = variable_name[len(CATALOG_PREFIX) :]
+ catalog_name, separator, property_name = remainder.partition("__")
+ if not separator:
+ continue
+
+ catalog_name = _normalize_key(catalog_name)
+ property_name = _normalize_key(property_name)
+ if (
+ catalog_name not in merged
+ or property_name not in CONFIDENCE_PROPERTIES
+ ):
+ continue
+ merged[catalog_name][property_name] = value
+
+ return merged
+
+
+def _lowercase_keys(value: object) -> object:
+ if isinstance(value, dict):
+ lowered: dict[str, object] = {}
+ for key, nested_value in value.items():
+ if not isinstance(key, str):
+ raise TypeError("PyIceberg configuration keys must be strings")
+ lowered[key.lower()] = _lowercase_keys(nested_value)
+ return lowered
+ if isinstance(value, list):
+ return [_lowercase_keys(item) for item in value]
+ return value
+
+
+def _catalogs_from_file_config(config: object) -> CatalogConfigurations:
+ if not isinstance(config, Mapping):
+ raise TypeError("PyIceberg configuration must be an object")
+
+ catalogs = config.get("catalog", {})
+ if not isinstance(catalogs, Mapping):
+ raise TypeError("PyIceberg catalog configuration must be an object")
+
+ result: dict[str, CatalogConfiguration] = {}
+ for name, catalog_config in catalogs.items():
+ if not isinstance(name, str) or not isinstance(
+ catalog_config, Mapping
+ ):
+ raise TypeError(
+ "PyIceberg catalog entries must map names to objects"
+ )
+ result[name] = cast(CatalogConfiguration, catalog_config)
+ return result
+
+
+def _iceberg_source(
+ *,
+ catalog_name: str,
+ origins: tuple[DetectedDataSourceOrigin, ...],
+ configuration: Sequence[DetectedDataSourceConfiguration],
+) -> DetectedDataSource:
+ return DetectedDataSource(
+ id=f"pyiceberg-{catalog_name}",
+ integration="pyiceberg",
+ category="catalog",
+ display_name=f"PyIceberg ({catalog_name})",
+ confidence="high",
+ origins=origins,
+ configuration=tuple(configuration),
+ code="\n".join(
+ [
+ "from pyiceberg.catalog import load_catalog",
+ "",
+ f"catalog = load_catalog({json.dumps(catalog_name)})",
+ ]
+ ),
+ )
+
+
+def _merge_catalogs(
+ sources: list[DetectedDataSource],
+) -> list[DetectedDataSource]:
+ by_id: dict[str, DetectedDataSource] = {}
+
+ for source in sources:
+ existing = by_id.get(source.id)
+ if existing is None:
+ by_id[source.id] = source
+ continue
+
+ by_id[source.id] = DetectedDataSource(
+ id=existing.id,
+ integration=existing.integration,
+ category=existing.category,
+ display_name=existing.display_name,
+ confidence=existing.confidence,
+ origins=tuple(dict.fromkeys((*existing.origins, *source.origins))),
+ configuration=tuple(
+ dict.fromkeys((*existing.configuration, *source.configuration))
+ ),
+ code=existing.code,
+ )
+
+ return list(by_id.values())
+
+
+def _normalize_key(value: str) -> str:
+ return value.lower().replace("__", ".").replace("_", "-")
+
+
+def _infer_catalog_type(
+ configuration: CatalogConfiguration,
+) -> str | None:
+ explicit_type = configuration.get("type")
+ if isinstance(explicit_type, str) and explicit_type.strip():
+ return _format_catalog_type(explicit_type)
+
+ catalog_impl = configuration.get("py-catalog-impl")
+ if isinstance(catalog_impl, str) and catalog_impl.strip():
+ return "Custom"
+
+ uri = configuration.get("uri")
+ if not isinstance(uri, str) or not uri.strip():
+ return None
+ normalized_uri = uri.lower()
+ if normalized_uri.startswith(("http://", "https://")):
+ return "REST"
+ if normalized_uri.startswith("thrift://"):
+ return "Hive"
+ if normalized_uri.startswith(("sqlite:", "postgresql")):
+ return "SQL"
+ return None
+
+
+def _format_catalog_type(value: str) -> str | None:
+ return CATALOG_TYPE_LABELS.get(value.strip().lower())
+
+
+PYICEBERG_PLUGIN = DiscoveryPlugin(id="pyiceberg", discover=discover)
diff --git a/marimo/_data/data_source_discovery/plugins/pyspark.py b/marimo/_data/data_source_discovery/plugins/pyspark.py
new file mode 100644
index 0000000000..22a7016104
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/pyspark.py
@@ -0,0 +1,42 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+ environment_variable,
+ has_value,
+)
+from marimo._data.data_source_discovery.models import DetectedDataSource
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+
+def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
+ if not has_value(context.environment, "SPARK_REMOTE"):
+ return []
+
+ return [
+ DetectedDataSource(
+ id="pyspark-connect-environment",
+ integration="pyspark",
+ category="database",
+ display_name="PySpark (Spark Connect)",
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=(environment_variable("Remote", "SPARK_REMOTE"),),
+ code="""\
+import os
+import ibis
+from pyspark.sql import SparkSession
+
+session = SparkSession.builder.remote(
+ os.environ["SPARK_REMOTE"]
+).getOrCreate()
+con = ibis.pyspark.connect(session)""",
+ )
+ ]
+
+
+PYSPARK_PLUGIN = DiscoveryPlugin(id="pyspark", discover=discover)
diff --git a/marimo/_data/data_source_discovery/plugins/trino.py b/marimo/_data/data_source_discovery/plugins/trino.py
new file mode 100644
index 0000000000..b1fc360d4e
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/trino.py
@@ -0,0 +1,104 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+ environment_variable,
+ has_all,
+ has_value,
+ is_valid_port,
+)
+from marimo._data.data_source_discovery.models import DetectedDataSource
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+REQUIRED = ("TRINO_HOST", "TRINO_USER", "TRINO_CATALOG")
+
+
+def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
+ environment = context.environment
+ if not has_all(environment, REQUIRED):
+ return []
+
+ has_valid_port = is_valid_port(environment.get("TRINO_PORT"))
+ configuration = [environment_variable("Host", "TRINO_HOST")]
+ if has_valid_port:
+ configuration.append(environment_variable("Port", "TRINO_PORT"))
+ configuration.append(environment_variable("Username", "TRINO_USER"))
+ if has_value(environment, "TRINO_PASSWORD"):
+ configuration.append(
+ environment_variable("Password", "TRINO_PASSWORD")
+ )
+ configuration.append(environment_variable("Catalog", "TRINO_CATALOG"))
+ if has_value(environment, "TRINO_SCHEMA"):
+ configuration.append(environment_variable("Schema", "TRINO_SCHEMA"))
+
+ database = ['database = os.environ["TRINO_CATALOG"]']
+ if has_value(environment, "TRINO_SCHEMA"):
+ database.append('database += "/" + os.environ["TRINO_SCHEMA"]')
+ has_password = has_value(environment, "TRINO_PASSWORD")
+ authentication = (
+ [
+ "connect_args = {",
+ ' "auth": BasicAuthentication(',
+ ' os.environ["TRINO_USER"],',
+ ' os.environ["TRINO_PASSWORD"],',
+ " ),",
+ ' "http_scheme": "https",',
+ "}",
+ "",
+ ]
+ if has_password
+ else []
+ )
+ port = (
+ ' port=int(os.environ["TRINO_PORT"]),'
+ if has_valid_port
+ else " port=8080,"
+ )
+ code = "\n".join(
+ [
+ "import os",
+ "import sqlalchemy",
+ "import trino.sqlalchemy",
+ *(
+ ["from trino.auth import BasicAuthentication"]
+ if has_password
+ else []
+ ),
+ "",
+ *authentication,
+ *database,
+ "TRINO_URL = sqlalchemy.URL.create(",
+ ' "trino",',
+ ' username=os.environ["TRINO_USER"],',
+ ' host=os.environ["TRINO_HOST"],',
+ port,
+ " database=database,",
+ ")",
+ (
+ "engine = sqlalchemy.create_engine("
+ "TRINO_URL, connect_args=connect_args)"
+ if has_password
+ else "engine = sqlalchemy.create_engine(TRINO_URL)"
+ ),
+ ]
+ )
+
+ return [
+ DetectedDataSource(
+ id="trino-environment",
+ integration="trino",
+ category="database",
+ display_name="Trino",
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=tuple(configuration),
+ code=code,
+ )
+ ]
+
+
+TRINO_PLUGIN = DiscoveryPlugin(id="trino", discover=discover)
diff --git a/marimo/_data/data_source_discovery/types.py b/marimo/_data/data_source_discovery/types.py
new file mode 100644
index 0000000000..f72777930f
--- /dev/null
+++ b/marimo/_data/data_source_discovery/types.py
@@ -0,0 +1,28 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from marimo._data.data_source_discovery.models import DetectedDataSource
+
+if TYPE_CHECKING:
+ from collections.abc import Callable, Mapping, Sequence
+
+
+@dataclass(frozen=True)
+class DiscoveryContext:
+ """Kernel-local inputs available to datasource discovery plugins."""
+
+ environment: Mapping[str, str]
+
+
+@dataclass(frozen=True)
+class DiscoveryPlugin:
+ """A self-contained, independently testable datasource detector."""
+
+ id: str
+ discover: Callable[
+ [DiscoveryContext],
+ Sequence[DetectedDataSource],
+ ]
diff --git a/marimo/_internal/commands.py b/marimo/_internal/commands.py
index 5a2f2ef4e7..642bec9f70 100644
--- a/marimo/_internal/commands.py
+++ b/marimo/_internal/commands.py
@@ -10,6 +10,7 @@ from marimo._runtime.commands import (
CreateNotebookCommand,
DebugCellCommand,
DeleteCellCommand,
+ DiscoverDataSourcesCommand,
ExecuteCellCommand,
ExecuteCellsCommand,
ExecuteScratchpadCommand,
@@ -50,6 +51,7 @@ __all__ = [
"CreateNotebookCommand",
"DebugCellCommand",
"DeleteCellCommand",
+ "DiscoverDataSourcesCommand",
"ExecuteCellCommand",
"ExecuteCellsCommand",
"ExecuteScratchpadCommand",
diff --git a/marimo/_lint/rules/runtime/reusable_definition_order.py b/marimo/_lint/rules/runtime/reusable_definition_order.py
index a75a2ead01..87c747f226 100644
--- a/marimo/_lint/rules/runtime/reusable_definition_order.py
+++ b/marimo/_lint/rules/runtime/reusable_definition_order.py
@@ -326,5 +326,5 @@ class ReusableDefinitionOrderRule(UnsafeFixRule):
@staticmethod
def _get_definition_name(status: TopLevelStatus) -> str:
if status.defs:
- return sorted(status.defs)[0]
+ return min(status.defs)
return status.name
diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.py
index be0acb7344..72dd4415fc 100644
--- a/marimo/_messaging/notification.py
+++ b/marimo/_messaging/notification.py
@@ -19,6 +19,7 @@ from marimo._data._external_storage.models import (
StorageEntry,
StorageNamespace,
)
+from marimo._data.data_source_discovery import DetectedDataSource
from marimo._data.models import (
ColumnStats,
DataSourceConnection,
@@ -751,6 +752,21 @@ class DataSourceConnectionsNotification(
connections: list[DataSourceConnection]
+class DataSourceDiscoveryResultNotification(
+ Notification, tag="data-source-discovery-result"
+):
+ """High-confidence datasource connections discovered by the kernel.
+
+ Attributes:
+ request_id: Request ID this responds to.
+ sources: Detected datasource connection configurations.
+ """
+
+ name: ClassVar[str] = "data-source-discovery-result"
+ request_id: RequestId
+ sources: list[DetectedDataSource]
+
+
class StorageNamespacesNotification(Notification, tag="storage-namespaces"):
"""Available storage namespaces for storage inspector.
@@ -990,6 +1006,7 @@ NotificationMessage = (
| SQLTableListPreviewNotification
| SQLSchemaListPreviewNotification
| DataSourceConnectionsNotification
+ | DataSourceDiscoveryResultNotification
| ValidateSQLResultNotification
# Storage
| StorageNamespacesNotification
diff --git a/marimo/_runtime/callbacks/datasets.py b/marimo/_runtime/callbacks/datasets.py
index fb255c4fed..4f32a96336 100644
--- a/marimo/_runtime/callbacks/datasets.py
+++ b/marimo/_runtime/callbacks/datasets.py
@@ -1,9 +1,11 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
+import os
from typing import TYPE_CHECKING, Any, cast
from marimo import _loggers
+from marimo._data.data_source_discovery import discover_data_sources
from marimo._data.preview_column import (
get_column_preview_for_dataframe,
get_column_preview_for_duckdb,
@@ -11,6 +13,7 @@ from marimo._data.preview_column import (
from marimo._messaging.notification import (
DataColumnPreviewNotification,
DataSourceConnectionsNotification,
+ DataSourceDiscoveryResultNotification,
SQLDatabaseMetadata,
SQLMetadata,
SQLSchemaListPreviewNotification,
@@ -19,6 +22,7 @@ from marimo._messaging.notification import (
)
from marimo._messaging.notification_utils import broadcast_notification
from marimo._runtime.commands import (
+ DiscoverDataSourcesCommand,
ListDataSourceConnectionCommand,
ListSQLSchemasCommand,
ListSQLTablesCommand,
@@ -53,6 +57,17 @@ class DatasetCallbacks:
ListDataSourceConnectionCommand,
self.preview_datasource_connection,
)
+ router.register(DiscoverDataSourcesCommand, self.discover_data_sources)
+
+ async def discover_data_sources(
+ self, request: DiscoverDataSourcesCommand
+ ) -> None:
+ broadcast_notification(
+ DataSourceDiscoveryResultNotification(
+ request_id=request.request_id,
+ sources=discover_data_sources(os.environ),
+ )
+ )
def get_engine_catalog(
self, variable_name: str
diff --git a/marimo/_runtime/commands.py b/marimo/_runtime/commands.py
index 1656ad49c8..3c610c59fd 100644
--- a/marimo/_runtime/commands.py
+++ b/marimo/_runtime/commands.py
@@ -817,6 +817,16 @@ class StorageDownloadCommand(Command):
preview: bool = False
+class DiscoverDataSourcesCommand(Command):
+ """Discover datasource connections from the live kernel environment and configuration.
+
+ Attributes:
+ request_id: Unique identifier for this request.
+ """
+
+ request_id: RequestId
+
+
class ListSecretKeysCommand(Command):
"""List available secret keys.
@@ -955,6 +965,7 @@ CommandMessage = (
| ListSQLSchemasCommand
| ValidateSQLCommand
| ListDataSourceConnectionCommand
+ | DiscoverDataSourcesCommand
# Storage operations
| StorageListEntriesCommand
| StorageDownloadCommand
diff --git a/marimo/_schemas/generated/notifications.yaml b/marimo/_schemas/generated/notifications.yaml
index 996064b1c0..3e1b56c2fd 100644
--- a/marimo/_schemas/generated/notifications.yaml
+++ b/marimo/_schemas/generated/notifications.yaml
@@ -523,6 +523,26 @@ components:
- connections
title: DataSourceConnectionsNotification
type: object
+ DataSourceDiscoveryResultNotification:
+ description: "High-confidence datasource connections discovered by the kernel.\n\
+ \nAttributes:\n request_id: Request ID this responds to.\n sources:\
+ \ Detected datasource connection configurations."
+ properties:
+ op:
+ enum:
+ - data-source-discovery-result
+ request_id:
+ type: string
+ sources:
+ items:
+ $ref: '#/components/schemas/DetectedDataSource'
+ type: array
+ required:
+ - op
+ - request_id
+ - sources
+ title: DataSourceDiscoveryResultNotification
+ type: object
DataTable:
description: "Represents a data table.\n\nAttributes:\n source_type (DataTableSource):\
\ Type of data source ('local', 'duckdb', 'connection').\n source (str):\
@@ -677,6 +697,89 @@ components:
- tables
title: DatasetsNotification
type: object
+ DetectedDataSource:
+ description: A secret-free datasource suggestion produced by the kernel.
+ properties:
+ category:
+ enum:
+ - catalog
+ - database
+ - object-storage
+ code:
+ type: string
+ confidence:
+ enum:
+ - high
+ - medium
+ configuration:
+ items:
+ $ref: '#/components/schemas/DetectedDataSourceConfiguration'
+ type: array
+ displayName:
+ type: string
+ id:
+ type: string
+ integration:
+ type: string
+ origins:
+ items:
+ $ref: '#/components/schemas/DetectedDataSourceOrigin'
+ type: array
+ required:
+ - id
+ - integration
+ - category
+ - displayName
+ - confidence
+ - origins
+ - configuration
+ - code
+ title: DetectedDataSource
+ type: object
+ DetectedDataSourceConfiguration:
+ properties:
+ field:
+ type: string
+ value:
+ anyOf:
+ - $ref: '#/components/schemas/EnvironmentVariableDiscoveryValue'
+ - $ref: '#/components/schemas/SafeLiteralDiscoveryValue'
+ discriminator:
+ mapping:
+ environment-variable: '#/components/schemas/EnvironmentVariableDiscoveryValue'
+ safe-literal: '#/components/schemas/SafeLiteralDiscoveryValue'
+ propertyName: kind
+ required:
+ - field
+ - value
+ title: DetectedDataSourceConfiguration
+ type: object
+ DetectedDataSourceOrigin:
+ properties:
+ label:
+ type: string
+ type:
+ enum:
+ - configuration
+ - environment
+ required:
+ - type
+ - label
+ title: DetectedDataSourceOrigin
+ type: object
+ EnvironmentVariableDiscoveryValue:
+ description: A reference to an environment variable, never its value.
+ properties:
+ kind:
+ enum:
+ - environment-variable
+ name:
+ type: string
+ required:
+ - kind
+ - name
+ title: EnvironmentVariableDiscoveryValue
+ type: object
FocusCellNotification:
description: "Focuses a cell (kiosk mode).\n\n Attributes:\n cell_id:\
\ Cell to focus."
@@ -1364,6 +1467,19 @@ components:
- table
title: SQLTablePreviewNotification
type: object
+ SafeLiteralDiscoveryValue:
+ description: Non-sensitive metadata that is safe to send to the frontend.
+ properties:
+ kind:
+ enum:
+ - safe-literal
+ value:
+ type: string
+ required:
+ - kind
+ - value
+ title: SafeLiteralDiscoveryValue
+ type: object
Schema:
properties:
name:
diff --git a/marimo/_server/api/endpoints/datasources.py b/marimo/_server/api/endpoints/datasources.py
index 1136b0b192..d5a9f3dc71 100644
--- a/marimo/_server/api/endpoints/datasources.py
+++ b/marimo/_server/api/endpoints/datasources.py
@@ -9,6 +9,7 @@ from marimo import _loggers
from marimo._server.api.utils import dispatch_control_request
from marimo._server.models.models import (
BaseResponse,
+ DiscoverDataSourcesRequest,
ListDataSourceConnectionRequest,
ListSQLSchemasRequest,
ListSQLTablesRequest,
@@ -26,6 +27,33 @@ LOGGER = _loggers.marimo_logger()
router = APIRouter()
[email protected]("/discover")
+@requires("edit")
+async def discover_data_sources(request: Request) -> BaseResponse:
+ """
+ parameters:
+ - in: header
+ name: Marimo-Session-Id
+ schema:
+ type: string
+ required: true
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/DiscoverDataSourcesRequest"
+ responses:
+ 200:
+ description: Discover datasource connections
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/SuccessResponse"
+ """
+ return await dispatch_control_request(request, DiscoverDataSourcesRequest)
+
+
@router.post("/preview_column")
@requires("edit")
async def preview_column(
diff --git a/marimo/_server/models/models.py b/marimo/_server/models/models.py
index 5d4d9a1c95..9f25dec41c 100644
--- a/marimo/_server/models/models.py
+++ b/marimo/_server/models/models.py
@@ -12,6 +12,7 @@ from marimo._runtime.commands import (
CodeCompletionCommand,
DebugCellCommand,
DeleteCellCommand,
+ DiscoverDataSourcesCommand,
ExecuteCellsCommand,
ExecuteScratchpadCommand,
GetCacheInfoCommand,
@@ -41,6 +42,11 @@ class ListSecretKeysRequest(ListSecretKeysCommand, tag=False):
return ListSecretKeysCommand(request_id=self.request_id)
+class DiscoverDataSourcesRequest(DiscoverDataSourcesCommand, tag=False):
+ def as_command(self) -> DiscoverDataSourcesCommand:
+ return DiscoverDataSourcesCommand(request_id=self.request_id)
+
+
class ClearCacheRequest(ClearCacheCommand, tag=False):
def as_command(self) -> ClearCacheCommand:
return ClearCacheCommand()
diff --git a/marimo/_session/capabilities.py b/marimo/_session/capabilities.py
index f9e08f2398..3e0502db6a 100644
--- a/marimo/_session/capabilities.py
+++ b/marimo/_session/capabilities.py
@@ -48,6 +48,7 @@ _READ_COMMANDS: frozenset[type] = frozenset(
commands.ListSQLSchemasCommand,
commands.ValidateSQLCommand,
commands.ListDataSourceConnectionCommand,
+ commands.DiscoverDataSourcesCommand,
commands.StorageListEntriesCommand,
commands.StorageDownloadCommand,
commands.ListSecretKeysCommand,
diff --git a/marimo/_utils/platform.py b/marimo/_utils/platform.py
index 98440b6fa2..c45818fff5 100644
--- a/marimo/_utils/platform.py
+++ b/marimo/_utils/platform.py
@@ -41,10 +41,12 @@ def check_shared_memory_available() -> tuple[bool, str]:
except OSError as e:
return (
False,
- f"Unable to create shared memory: {e}. "
- "This can happen in restricted environments like some Docker "
- "containers or when /dev/shm is not available or has insufficient "
- "space.",
+ (
+ f"Unable to create shared memory: {e}. "
+ "This can happen in restricted environments like some Docker "
+ "containers or when /dev/shm is not available or has "
+ "insufficient space."
+ ),
)
except Exception as e:
return False, f"Unexpected error checking shared memory: {e}"
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index b750db0d9c..3f440e5301 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -1154,6 +1154,26 @@ components:
- connections
title: DataSourceConnectionsNotification
type: object
+ DataSourceDiscoveryResultNotification:
+ description: "High-confidence datasource connections discovered by the kernel.\n\
+ \n Attributes:\n request_id: Request ID this responds to.\n \
+ \ sources: Detected datasource connection configurations."
+ properties:
+ op:
+ enum:
+ - data-source-discovery-result
+ request_id:
+ type: string
+ sources:
+ items:
+ $ref: '#/components/schemas/DetectedDataSource'
+ type: array
+ required:
+ - op
+ - request_id
+ - sources
+ title: DataSourceDiscoveryResultNotification
+ type: object
DataTable:
description: "Represents a data table.\n\nAttributes:\n source_type (DataTableSource):\
\ Type of data source ('local', 'duckdb', 'connection').\n source (str):\
@@ -1459,6 +1479,76 @@ components:
- tree
title: DependencyTreeResponse
type: object
+ DetectedDataSource:
+ description: A secret-free datasource suggestion produced by the kernel.
+ properties:
+ category:
+ enum:
+ - catalog
+ - database
+ - object-storage
+ code:
+ type: string
+ confidence:
+ enum:
+ - high
+ - medium
+ configuration:
+ items:
+ $ref: '#/components/schemas/DetectedDataSourceConfiguration'
+ type: array
+ displayName:
+ type: string
+ id:
+ type: string
+ integration:
+ type: string
+ origins:
+ items:
+ $ref: '#/components/schemas/DetectedDataSourceOrigin'
+ type: array
+ required:
+ - id
+ - integration
+ - category
+ - displayName
+ - confidence
+ - origins
+ - configuration
+ - code
+ title: DetectedDataSource
+ type: object
+ DetectedDataSourceConfiguration:
+ properties:
+ field:
+ type: string
+ value:
+ anyOf:
+ - $ref: '#/components/schemas/EnvironmentVariableDiscoveryValue'
+ - $ref: '#/components/schemas/SafeLiteralDiscoveryValue'
+ discriminator:
+ mapping:
+ environment-variable: '#/components/schemas/EnvironmentVariableDiscoveryValue'
+ safe-literal: '#/components/schemas/SafeLiteralDiscoveryValue'
+ propertyName: kind
+ required:
+ - field
+ - value
+ title: DetectedDataSourceConfiguration
+ type: object
+ DetectedDataSourceOrigin:
+ properties:
+ label:
+ type: string
+ type:
+ enum:
+ - configuration
+ - environment
+ required:
+ - type
+ - label
+ title: DetectedDataSourceOrigin
+ type: object
DiagnosticsConfig:
description: "Configuration options for diagnostics.\n\n **Keys.**\n\n \
\ - `enabled`: if `True`, diagnostics will be shown in the editor\n -\
@@ -1471,6 +1561,29 @@ components:
required: []
title: DiagnosticsConfig
type: object
+ DiscoverDataSourcesCommand:
+ description: "Discover datasource connections from the live kernel environment\
+ \ and configuration.\n\n Attributes:\n request_id: Unique identifier\
+ \ for this request."
+ properties:
+ requestId:
+ $ref: '#/components/schemas/RequestId'
+ type:
+ enum:
+ - discover-data-sources
+ required:
+ - type
+ - requestId
+ title: DiscoverDataSourcesCommand
+ type: object
+ DiscoverDataSourcesRequest:
+ properties:
+ requestId:
+ $ref: '#/components/schemas/RequestId'
+ required:
+ - requestId
+ title: DiscoverDataSourcesRequest
+ type: object
DisplayConfig:
description: "Configuration for display.\n\n **Keys.**\n\n - `theme`:\
\ `\"light\"`, `\"dark\"`, or `\"system\"`\n - `code_editor_font_size`:\
@@ -1529,6 +1642,19 @@ components:
- theme
title: DisplayConfig
type: object
+ EnvironmentVariableDiscoveryValue:
+ description: A reference to an environment variable, never its value.
+ properties:
+ kind:
+ enum:
+ - environment-variable
+ name:
+ type: string
+ required:
+ - kind
+ - name
+ title: EnvironmentVariableDiscoveryValue
+ type: object
EsmSpec:
description: "Where the frontend imports a widget's ESM from, and which version.\n\
\n Specs travel only on kernel-authored notifications, never in model\n\
@@ -2765,6 +2891,7 @@ components:
- $ref: '#/components/schemas/ListSQLSchemasCommand'
- $ref: '#/components/schemas/ValidateSQLCommand'
- $ref: '#/components/schemas/ListDataSourceConnectionCommand'
+ - $ref: '#/components/schemas/DiscoverDataSourcesCommand'
- $ref: '#/components/schemas/StorageListEntriesCommand'
- $ref: '#/components/schemas/StorageDownloadCommand'
- $ref: '#/components/schemas/ListSecretKeysCommand'
@@ -2779,6 +2906,7 @@ components:
create-notebook: '#/components/schemas/CreateNotebookCommand'
debug-cell: '#/components/schemas/DebugCellCommand'
delete-cell: '#/components/schemas/DeleteCellCommand'
+ discover-data-sources: '#/components/schemas/DiscoverDataSourcesCommand'
execute-cells: '#/components/schemas/ExecuteCellsCommand'
execute-scratchpad: '#/components/schemas/ExecuteScratchpadCommand'
execute-stale-cells: '#/components/schemas/ExecuteStaleCellsCommand'
@@ -2876,6 +3004,7 @@ components:
- $ref: '#/components/schemas/SQLTableListPreviewNotification'
- $ref: '#/components/schemas/SQLSchemaListPreviewNotification'
- $ref: '#/components/schemas/DataSourceConnectionsNotification'
+ - $ref: '#/components/schemas/DataSourceDiscoveryResultNotification'
- $ref: '#/components/schemas/ValidateSQLResultNotification'
- $ref: '#/components/schemas/StorageNamespacesNotification'
- $ref: '#/components/schemas/StorageEntriesNotification'
@@ -2900,6 +3029,7 @@ components:
consumer-capabilities: '#/components/schemas/ConsumerCapabilitiesNotification'
data-column-preview: '#/components/schemas/DataColumnPreviewNotification'
data-source-connections: '#/components/schemas/DataSourceConnectionsNotification'
+ data-source-discovery-result: '#/components/schemas/DataSourceDiscoveryResultNotification'
datasets: '#/components/schemas/DatasetsNotification'
focus-cell: '#/components/schemas/FocusCellNotification'
function-call-result: '#/components/schemas/FunctionCallResultNotification'
@@ -4618,6 +4748,19 @@ components:
- table
title: SQLTablePreviewNotification
type: object
+ SafeLiteralDiscoveryValue:
+ description: Non-sensitive metadata that is safe to send to the frontend.
+ properties:
+ kind:
+ enum:
+ - safe-literal
+ value:
+ type: string
+ required:
+ - kind
+ - value
+ title: SafeLiteralDiscoveryValue
+ type: object
SaveAppConfigurationRequest:
properties:
config:
@@ -6172,6 +6315,27 @@ paths:
schema:
$ref: '#/components/schemas/SuccessResponse'
description: Get cache statistics
+ /api/datasources/discover:
+ post:
+ parameters:
+ - in: header
+ name: Marimo-Session-Id
+ required: true
+ schema:
+ type: string
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/DiscoverDataSourcesRequest'
+ required: true
+ responses:
+ 200:
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/SuccessResponse'
+ description: Discover datasource connections
/api/datasources/preview_column:
post:
parameters:
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index bee1b91950..f3edbb114a 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -366,6 +366,47 @@ export interface paths {
patch?: never;
trace?: never;
};
+ "/api/datasources/discover": {
+ parameters: {
+ query?: never;
+ header?: never;
+ path?: never;
+ cookie?: never;
+ };
+ get?: never;
+ put?: never;
+ post: {
+ parameters: {
+ query?: never;
+ header: {
+ "Marimo-Session-Id": string;
+ };
+ path?: never;
+ cookie?: never;
+ };
+ requestBody: {
+ content: {
+ "application/json": components["schemas"]["DiscoverDataSourcesRequest"];
+ };
+ };
+ responses: {
+ /** @description Discover datasource connections */
+ 200: {
+ headers: {
+ [name: string]: unknown;
+ };
+ content: {
+ "application/json": components["schemas"]["SuccessResponse"];
+ };
+ };
+ };
+ };
+ delete?: never;
+ options?: never;
+ head?: never;
+ patch?: never;
+ trace?: never;
+ };
"/api/datasources/preview_column": {
parameters: {
query?: never;
@@ -4294,6 +4335,20 @@ export interface components {
/** @enum {unknown} */
op: "data-source-connections";
};
+ /**
+ * DataSourceDiscoveryResultNotification
+ * @description High-confidence datasource connections discovered by the kernel.
+ *
+ * Attributes:
+ * request_id: Request ID this responds to.
+ * sources: Detected datasource connection configurations.
+ */
+ DataSourceDiscoveryResultNotification: {
+ /** @enum {unknown} */
+ op: "data-source-discovery-result";
+ request_id: string;
+ sources: components["schemas"]["DetectedDataSource"][];
+ };
/**
* DataTable
* @description Represents a data table.
@@ -4479,6 +4534,35 @@ export interface components {
DependencyTreeResponse: {
tree: null | components["schemas"]["DependencyTreeNode"];
};
+ /**
+ * DetectedDataSource
+ * @description A secret-free datasource suggestion produced by the kernel.
+ */
+ DetectedDataSource: {
+ /** @enum {unknown} */
+ category: "catalog" | "database" | "object-storage";
+ code: string;
+ /** @enum {unknown} */
+ confidence: "high" | "medium";
+ configuration: components["schemas"]["DetectedDataSourceConfiguration"][];
+ displayName: string;
+ id: string;
+ integration: string;
+ origins: components["schemas"]["DetectedDataSourceOrigin"][];
+ };
+ /** DetectedDataSourceConfiguration */
+ DetectedDataSourceConfiguration: {
+ field: string;
+ value:
+ | components["schemas"]["EnvironmentVariableDiscoveryValue"]
+ | components["schemas"]["SafeLiteralDiscoveryValue"];
+ };
+ /** DetectedDataSourceOrigin */
+ DetectedDataSourceOrigin: {
+ label: string;
+ /** @enum {unknown} */
+ type: "configuration" | "environment";
+ };
/**
* DiagnosticsConfig
* @description Configuration options for diagnostics.
@@ -4492,6 +4576,22 @@ export interface components {
enabled?: boolean;
sql_linter?: boolean;
};
+ /**
+ * DiscoverDataSourcesCommand
+ * @description Discover datasource connections from the live kernel environment and configuration.
+ *
+ * Attributes:
+ * request_id: Unique identifier for this request.
+ */
+ DiscoverDataSourcesCommand: {
+ requestId: components["schemas"]["RequestId"];
+ /** @enum {unknown} */
+ type: "discover-data-sources";
+ };
+ /** DiscoverDataSourcesRequest */
+ DiscoverDataSourcesRequest: {
+ requestId: components["schemas"]["RequestId"];
+ };
/**
* DisplayConfig
* @description Configuration for display.
@@ -4524,6 +4624,15 @@ export interface components {
/** @enum {unknown} */
theme: "dark" | "light" | "system";
};
+ /**
+ * EnvironmentVariableDiscoveryValue
+ * @description A reference to an environment variable, never its value.
+ */
+ EnvironmentVariableDiscoveryValue: {
+ /** @enum {unknown} */
+ kind: "environment-variable";
+ name: string;
+ };
/**
* EsmSpec
* @description Where the frontend imports a widget's ESM from, and which version.
@@ -5289,6 +5398,7 @@ export interface components {
| components["schemas"]["ListSQLSchemasCommand"]
| components["schemas"]["ValidateSQLCommand"]
| components["schemas"]["ListDataSourceConnectionCommand"]
+ | components["schemas"]["DiscoverDataSourcesCommand"]
| components["schemas"]["StorageListEntriesCommand"]
| components["schemas"]["StorageDownloadCommand"]
| components["schemas"]["ListSecretKeysCommand"]
@@ -5350,6 +5460,7 @@ export interface components {
| components["schemas"]["SQLTableListPreviewNotification"]
| components["schemas"]["SQLSchemaListPreviewNotification"]
| components["schemas"]["DataSourceConnectionsNotification"]
+ | components["schemas"]["DataSourceDiscoveryResultNotification"]
| components["schemas"]["ValidateSQLResultNotification"]
| components["schemas"]["StorageNamespacesNotification"]
| components["schemas"]["StorageEntriesNotification"]
@@ -6423,6 +6534,15 @@ export interface components {
request_id: components["schemas"]["RequestId"];
table: null | components["schemas"]["DataTable"];
};
+ /**
+ * SafeLiteralDiscoveryValue
+ * @description Non-sensitive metadata that is safe to send to the frontend.
+ */
+ SafeLiteralDiscoveryValue: {
+ /** @enum {unknown} */
+ kind: "safe-literal";
+ value: string;
+ };
/** SaveAppConfigurationRequest */
SaveAppConfigurationRequest: {
config: Record<string, any>;
diff --git a/scripts/generate_schemas.py b/scripts/generate_schemas.py
index 3c54ab9fb0..c155b6d632 100755
--- a/scripts/generate_schemas.py
+++ b/scripts/generate_schemas.py
@@ -55,6 +55,7 @@ with app.setup:
o.SQLSchemaListPreviewNotification,
o.DataColumnPreviewNotification,
o.DataSourceConnectionsNotification,
+ o.DataSourceDiscoveryResultNotification,
o.ValidateSQLResultNotification,
o.QueryParamsSetNotification,
o.QueryParamsAppendNotification,
diff --git a/tests/_ast/test_visitor.py b/tests/_ast/test_visitor.py
index eb688f10ce..1a405578f1 100644
--- a/tests/_ast/test_visitor.py
+++ b/tests/_ast/test_visitor.py
@@ -1000,10 +1000,12 @@ def test_underscore_as_imports_are_cell_local() -> None:
("import a.b as _c", {f"{prefix}c"}),
("from a.b import _c as _d", {f"{prefix}d"}),
(
- "import marimo as _private\n"
- "@_private.cache\n"
- "def f(x):\n"
- " return _private.md('x')\n",
+ (
+ "import marimo as _private\n"
+ "@_private.cache\n"
+ "def f(x):\n"
+ " return _private.md('x')\n"
+ ),
{f"{prefix}private", "f"},
),
# `from x import y as _p` referenced in nested scopes (class base,
@@ -1014,10 +1016,12 @@ def test_underscore_as_imports_are_cell_local() -> None:
{f"{prefix}private", "C"},
),
(
- "from a.b import c as _also_private\n"
- "@_also_private.deco\n"
- "def f(x):\n"
- " return _also_private.g(x)\n",
+ (
+ "from a.b import c as _also_private\n"
+ "@_also_private.deco\n"
+ "def f(x):\n"
+ " return _also_private.g(x)\n"
+ ),
{f"{prefix}also_private", "f"},
),
]
@@ -1069,10 +1073,12 @@ def test_no_alias_underscore_import_refs_not_mangled() -> None:
{"_camera", "C"},
),
(
- "from a.b import _camera\n"
- "@_camera.deco\n"
- "def f(x):\n"
- " return _camera.g(x)\n",
+ (
+ "from a.b import _camera\n"
+ "@_camera.deco\n"
+ "def f(x):\n"
+ " return _camera.g(x)\n"
+ ),
{"_camera", "f"},
),
]
diff --git a/tests/_data/data_source_discovery/test_aws.py b/tests/_data/data_source_discovery/test_aws.py
new file mode 100644
index 0000000000..57b16beef0
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_aws.py
@@ -0,0 +1,128 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import msgspec
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery.models import (
+ EnvironmentVariableDiscoveryValue,
+)
+from marimo._data.data_source_discovery.plugins.aws import discover
+from marimo._data.data_source_discovery.types import DiscoveryContext
+
+
+def test_discovers_s3_compatible_environment() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "AWS_ACCESS_KEY_ID": "secret-access-key",
+ "AWS_SECRET_ACCESS_KEY": "secret-secret-key",
+ "AWS_ENDPOINT_URL_S3": "https://example.invalid",
+ "AWS_REGION": "us-east-1",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins == snapshot(
+ [
+ {
+ "id": "aws-s3-environment",
+ "integration": "aws",
+ "category": "object-storage",
+ "displayName": "S3-compatible storage",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "Access key ID",
+ "value": {
+ "kind": "environment-variable",
+ "name": "AWS_ACCESS_KEY_ID",
+ },
+ },
+ {
+ "field": "Secret access key",
+ "value": {
+ "kind": "environment-variable",
+ "name": "AWS_SECRET_ACCESS_KEY",
+ },
+ },
+ {
+ "field": "Region",
+ "value": {
+ "kind": "environment-variable",
+ "name": "AWS_REGION",
+ },
+ },
+ {
+ "field": "S3 endpoint",
+ "value": {
+ "kind": "environment-variable",
+ "name": "AWS_ENDPOINT_URL_S3",
+ },
+ },
+ ],
+ "code": """\
+import s3fs
+
+# Uses the standard AWS credential and endpoint environment variables.
+fs = s3fs.S3FileSystem()""",
+ }
+ ]
+ )
+ assert "secret-" not in repr(builtins)
+
+
+def test_discovers_profile_without_access_keys() -> None:
+ detected = discover(
+ DiscoveryContext(environment={"AWS_PROFILE": "secret-profile"})
+ )
+
+ assert msgspec.json.decode(
+ msgspec.json.encode(detected[0].configuration)
+ ) == snapshot(
+ [
+ {
+ "field": "Profile",
+ "value": {
+ "kind": "environment-variable",
+ "name": "AWS_PROFILE",
+ },
+ }
+ ]
+ )
+
+
+def test_ignores_incomplete_access_keys() -> None:
+ detected = discover(
+ DiscoveryContext(environment={"AWS_ACCESS_KEY_ID": "secret"})
+ )
+
+ assert detected == snapshot([])
+
+
+def test_ignores_empty_profile_and_endpoint() -> None:
+ assert discover(
+ DiscoveryContext(environment={"AWS_PROFILE": ""})
+ ) == snapshot([])
+
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "AWS_PROFILE": "secret-profile",
+ "AWS_ENDPOINT_URL_S3": "",
+ }
+ )
+ )
+ assert detected[0].display_name == snapshot("Amazon S3")
+ assert [
+ item.value.name
+ for item in detected[0].configuration
+ if isinstance(item.value, EnvironmentVariableDiscoveryValue)
+ ] == snapshot(["AWS_PROFILE"])
diff --git a/tests/_data/data_source_discovery/test_discover.py b/tests/_data/data_source_discovery/test_discover.py
new file mode 100644
index 0000000000..810e5119d8
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_discover.py
@@ -0,0 +1,181 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import ast
+import logging
+from io import StringIO
+from unittest.mock import patch
+
+import msgspec
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery import discover_data_sources
+from marimo._data.data_source_discovery.discover import LOGGER
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+)
+from marimo._data.data_source_discovery.models import DetectedDataSource
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+
+def test_isolates_failures_and_deduplicates() -> None:
+ source = DetectedDataSource(
+ id="shared",
+ integration="example",
+ category="database",
+ display_name="Shared",
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=(),
+ code="connection = shared()",
+ )
+ duplicate = DetectedDataSource(
+ id="shared",
+ integration="replacement",
+ category="database",
+ display_name="Replacement",
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=(),
+ code="connection = replacement()",
+ )
+
+ def fail(_context: DiscoveryContext) -> list[DetectedDataSource]:
+ raise RuntimeError("broken detector")
+
+ with patch(
+ "marimo._data.data_source_discovery.discover.LOGGER.warning"
+ ) as log_warning:
+ detected = discover_data_sources(
+ {},
+ plugins=(
+ DiscoveryPlugin(
+ id="first", discover=lambda _context: [source]
+ ),
+ DiscoveryPlugin(
+ id="duplicate", discover=lambda _context: [duplicate]
+ ),
+ DiscoveryPlugin(id="broken", discover=fail),
+ ),
+ )
+
+ assert detected == snapshot([source])
+ log_warning.assert_called_once_with(
+ "Datasource discovery plugin %s failed (%s)",
+ "broken",
+ "RuntimeError",
+ )
+
+
+def test_default_plugins_emit_valid_secret_free_suggestions() -> None:
+ environment = {
+ "PGHOST": "secret-pg-host",
+ "PGPORT": "61231",
+ "PGUSER": "secret-pg-user",
+ "PGPASSWORD": "secret-pg-password",
+ "PGDATABASE": "secret-pg-database",
+ "MYSQL_HOST": "secret-mysql-host",
+ "MYSQL_TCP_PORT": "61232",
+ "MYSQL_USER": "secret-mysql-user",
+ "MYSQL_PWD": "secret-mysql-password",
+ "MYSQL_DATABASE": "secret-mysql-database",
+ "AWS_PROFILE": "secret-aws-profile",
+ "TRINO_HOST": "secret-trino-host",
+ "TRINO_PORT": "61233",
+ "TRINO_USER": "secret-trino-user",
+ "TRINO_PASSWORD": "secret-trino-password",
+ "TRINO_CATALOG": "secret-trino-catalog",
+ "TRINO_SCHEMA": "secret-trino-schema",
+ "SPARK_REMOTE": "sc://secret-spark-host:61234",
+ "PYICEBERG_CATALOG__PROD__URI": "https://secret-iceberg.invalid",
+ }
+
+ with patch(
+ "marimo._data.data_source_discovery.plugins.pyiceberg."
+ "_load_resolved_catalogs",
+ return_value={
+ "resolved": {
+ "type": "rest",
+ "credential": "secret-resolved-credential",
+ }
+ },
+ ):
+ detected = discover_data_sources(environment)
+
+ assert [source.integration for source in detected] == snapshot(
+ [
+ "postgres",
+ "mysql",
+ "aws",
+ "trino",
+ "pyspark",
+ "pyiceberg",
+ "pyiceberg",
+ ]
+ )
+ payload = msgspec.json.encode(detected).decode()
+ for value in environment.values():
+ assert value not in payload
+ assert "secret-resolved-credential" not in payload
+ for source in detected:
+ ast.parse(source.code)
+
+
+def test_plugins_receive_an_immutable_environment_snapshot() -> None:
+ observed: list[str] = []
+
+ def mutate(context: DiscoveryContext) -> list[DetectedDataSource]:
+ context.environment["INJECTED"] = "value" # type: ignore[index]
+ return []
+
+ def observe(context: DiscoveryContext) -> list[DetectedDataSource]:
+ observed.extend(context.environment)
+ return []
+
+ with patch(
+ "marimo._data.data_source_discovery.discover.LOGGER.warning"
+ ) as log_warning:
+ discover_data_sources(
+ {"ORIGINAL": "value"},
+ plugins=(
+ DiscoveryPlugin(id="mutating", discover=mutate),
+ DiscoveryPlugin(id="observing", discover=observe),
+ ),
+ )
+
+ assert observed == snapshot(["ORIGINAL"])
+ log_warning.assert_called_once_with(
+ "Datasource discovery plugin %s failed (%s)",
+ "mutating",
+ "TypeError",
+ )
+
+
+def test_plugin_error_logs_never_include_exception_message() -> None:
+ secret = "TOP-SECRET-URI"
+
+ def fail(_context: DiscoveryContext) -> list[DetectedDataSource]:
+ raise ValueError(
+ "Incompatible configurations, merging dict with a value: "
+ f"uri, value: {secret}"
+ )
+
+ log_output = StringIO()
+ handler = logging.StreamHandler(log_output)
+ LOGGER.addHandler(handler)
+ try:
+ detected = discover_data_sources(
+ {},
+ plugins=(DiscoveryPlugin(id="pyiceberg", discover=fail),),
+ )
+ finally:
+ LOGGER.removeHandler(handler)
+ logs = log_output.getvalue()
+
+ assert detected == snapshot([])
+ assert secret not in logs
+ assert "ValueError" in logs
+ assert "pyiceberg" in logs
diff --git a/tests/_data/data_source_discovery/test_discovery_pyiceberg.py b/tests/_data/data_source_discovery/test_discovery_pyiceberg.py
new file mode 100644
index 0000000000..49523351d3
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_discovery_pyiceberg.py
@@ -0,0 +1,466 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import ast
+import os
+from typing import TYPE_CHECKING
+from unittest.mock import patch
+
+import msgspec
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery.models import (
+ SafeLiteralDiscoveryValue,
+)
+from marimo._data.data_source_discovery.plugins.pyiceberg import (
+ _load_resolved_catalogs,
+ discover,
+)
+from marimo._data.data_source_discovery.types import DiscoveryContext
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+
+def test_merges_environment_with_resolved_configuration() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PYICEBERG_CATALOG__PROD__URI": "https://secret.invalid",
+ "PYICEBERG_CATALOG__PROD__WAREHOUSE": "s3://secret",
+ }
+ ),
+ load_catalogs=lambda: {
+ "hive": {
+ "uri": "thrift://secret-host:9083",
+ "credential": "do-not-copy",
+ },
+ "prod": {
+ "type": "rest",
+ "credential": "do-not-copy",
+ },
+ },
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins == snapshot(
+ [
+ {
+ "id": "pyiceberg-prod",
+ "integration": "pyiceberg",
+ "category": "catalog",
+ "displayName": "PyIceberg (prod)",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ },
+ {
+ "type": "configuration",
+ "label": "Resolved PyIceberg configuration",
+ },
+ ],
+ "configuration": [
+ {
+ "field": "Catalog",
+ "value": {
+ "kind": "safe-literal",
+ "value": "prod",
+ },
+ },
+ {
+ "field": "uri",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PYICEBERG_CATALOG__PROD__URI",
+ },
+ },
+ {
+ "field": "warehouse",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PYICEBERG_CATALOG__PROD__WAREHOUSE",
+ },
+ },
+ {
+ "field": "Type",
+ "value": {
+ "kind": "safe-literal",
+ "value": "REST",
+ },
+ },
+ ],
+ "code": """\
+from pyiceberg.catalog import load_catalog
+
+catalog = load_catalog("prod")""",
+ },
+ {
+ "id": "pyiceberg-hive",
+ "integration": "pyiceberg",
+ "category": "catalog",
+ "displayName": "PyIceberg (hive)",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "configuration",
+ "label": "Resolved PyIceberg configuration",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "Catalog",
+ "value": {
+ "kind": "safe-literal",
+ "value": "hive",
+ },
+ },
+ {
+ "field": "Type",
+ "value": {
+ "kind": "safe-literal",
+ "value": "Hive",
+ },
+ },
+ ],
+ "code": """\
+from pyiceberg.catalog import load_catalog
+
+catalog = load_catalog("hive")""",
+ },
+ ]
+ )
+ assert "do-not-copy" not in repr(builtins)
+ assert "secret" not in repr(builtins)
+
+
+def test_ignores_low_confidence_catalogs() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={"PYICEBERG_CATALOG__PROD__WAREHOUSE": "s3://secret"}
+ ),
+ load_catalogs=lambda: {"prod": {"credential": "do-not-copy"}},
+ )
+
+ assert detected == snapshot([])
+
+
+def test_resolved_configuration_failure_preserves_environment_catalogs() -> (
+ None
+):
+ sentinel = "TOP-SECRET-CONFIGURATION"
+
+ def fail() -> dict[str, dict[str, str]]:
+ raise ValueError(sentinel)
+
+ with patch(
+ "marimo._data.data_source_discovery.plugins.pyiceberg.LOGGER.warning"
+ ) as log_warning:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PYICEBERG_CATALOG__PROD__URI": ("https://secret.invalid")
+ }
+ ),
+ load_catalogs=fail,
+ )
+
+ payload = msgspec.json.encode(detected).decode()
+ assert [source.id for source in detected] == snapshot(["pyiceberg-prod"])
+ assert sentinel not in payload
+ log_warning.assert_called_once_with(
+ "PyIceberg resolved configuration discovery failed (%s)",
+ "ValueError",
+ )
+ assert sentinel not in repr(log_warning.call_args_list)
+
+
+def test_rejects_unknown_explicit_catalog_type_without_leaking_it() -> None:
+ sentinel = "TOP-SECRET-TOKEN"
+ detected = discover(
+ DiscoveryContext(environment={}),
+ load_catalogs=lambda: {
+ "unknown": {
+ "type": sentinel,
+ "uri": "https://catalog.invalid",
+ }
+ },
+ )
+
+ payload = msgspec.json.encode(detected).decode()
+ assert detected == snapshot([])
+ assert sentinel.casefold() not in payload.casefold()
+
+
+def test_ignores_empty_and_malformed_environment_catalogs() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PYICEBERG_CATALOG__PROD__URI": "",
+ "PYICEBERG_CATALOG____URI": "https://secret.invalid",
+ "PYICEBERG_CATALOG__MISSING_PROPERTY": "secret",
+ "PYICEBERG_CATALOG__LOW__WAREHOUSE": "s3://secret",
+ }
+ ),
+ load_catalogs=dict,
+ )
+
+ assert detected == snapshot([])
+
+
+def test_resolved_catalog_type_inference_and_name_escaping() -> None:
+ detected = discover(
+ DiscoveryContext(environment={}),
+ load_catalogs=lambda: {
+ "blank": {
+ "type": " ",
+ "uri": "",
+ "py-catalog-impl": object(),
+ },
+ 'quoted"name': {
+ "type": " REST ",
+ "credential": "do-not-copy",
+ },
+ "sql": {"uri": "POSTGRESQL://secret.invalid/database"},
+ "custom": {"py-catalog-impl": "package.CustomCatalog"},
+ },
+ )
+
+ detected_types: list[tuple[str, str]] = []
+ for source in detected:
+ value = source.configuration[-1].value
+ assert isinstance(value, SafeLiteralDiscoveryValue)
+ detected_types.append((source.id, value.value))
+
+ assert detected_types == snapshot(
+ [
+ ("pyiceberg-custom", "Custom"),
+ ('pyiceberg-quoted"name', "REST"),
+ ("pyiceberg-sql", "SQL"),
+ ]
+ )
+ for source in detected:
+ ast.parse(source.code)
+ assert 'load_catalog("quoted\\"name")' in detected[1].code
+ assert "do-not-copy" not in repr(detected)
+ assert "secret.invalid" not in repr(detected)
+
+
+def test_supports_known_explicit_catalog_types() -> None:
+ detected = discover(
+ DiscoveryContext(environment={}),
+ load_catalogs=lambda: {
+ catalog_type: {"type": catalog_type}
+ for catalog_type in (
+ "rest",
+ "hive",
+ "glue",
+ "dynamodb",
+ "sql",
+ "in-memory",
+ "bigquery",
+ )
+ },
+ )
+
+ detected_types: list[tuple[str, str]] = []
+ for source in detected:
+ value = source.configuration[-1].value
+ assert isinstance(value, SafeLiteralDiscoveryValue)
+ detected_types.append((source.id, value.value))
+
+ assert detected_types == snapshot(
+ [
+ ("pyiceberg-bigquery", "BigQuery"),
+ ("pyiceberg-dynamodb", "DynamoDB"),
+ ("pyiceberg-glue", "Glue"),
+ ("pyiceberg-hive", "Hive"),
+ ("pyiceberg-in-memory", "In-memory"),
+ ("pyiceberg-rest", "REST"),
+ ("pyiceberg-sql", "SQL"),
+ ]
+ )
+
+
+def test_configuration_file_search_precedence(tmp_path: Path) -> None:
+ configured_home = tmp_path / "configured"
+ home_directory = tmp_path / "home"
+ current_directory = tmp_path / "cwd"
+ missing_home = tmp_path / "missing-home"
+ for directory in (
+ configured_home,
+ home_directory,
+ current_directory,
+ missing_home,
+ ):
+ directory.mkdir()
+
+ (configured_home / ".pyiceberg.yaml").write_text(
+ """\
+CATALOG:
+ configured:
+ TYPE: REST
+ TOKEN: TOP-SECRET-CONFIGURED
+""",
+ encoding="utf-8",
+ )
+ (home_directory / ".pyiceberg.yaml").write_text(
+ """\
+catalog:
+ home:
+ uri: thrift://TOP-SECRET-HOME
+""",
+ encoding="utf-8",
+ )
+ (current_directory / ".pyiceberg.yaml").write_text(
+ """\
+catalog:
+ current:
+ type: sql
+ credential: TOP-SECRET-CURRENT
+""",
+ encoding="utf-8",
+ )
+
+ with patch(
+ "marimo._data.data_source_discovery.plugins.pyiceberg."
+ "DependencyManager.pyiceberg.has",
+ return_value=False,
+ ):
+ from_configured_home = _load_resolved_catalogs(
+ environment={"PYICEBERG_HOME": str(configured_home)},
+ home_directory=home_directory,
+ current_directory=current_directory,
+ )
+ from_home = _load_resolved_catalogs(
+ environment={},
+ home_directory=home_directory,
+ current_directory=current_directory,
+ )
+ from_current_directory = _load_resolved_catalogs(
+ environment={},
+ home_directory=missing_home,
+ current_directory=current_directory,
+ )
+
+ assert {
+ "configured": list(from_configured_home),
+ "home": list(from_home),
+ "current": list(from_current_directory),
+ } == snapshot(
+ {
+ "configured": ["configured"],
+ "home": ["home"],
+ "current": ["current"],
+ }
+ )
+
+ detected = discover(
+ DiscoveryContext(environment={}),
+ load_catalogs=lambda: from_configured_home,
+ )
+ payload = msgspec.json.encode(detected).decode()
+ assert "TOP-SECRET-CONFIGURED" not in payload
+ assert detected[0].display_name == snapshot("PyIceberg (configured)")
+
+
+def test_discovery_uses_the_context_environment_snapshot(
+ tmp_path: Path,
+) -> None:
+ context_home = tmp_path / "context"
+ process_home = tmp_path / "process"
+ context_home.mkdir()
+ process_home.mkdir()
+ (context_home / ".pyiceberg.yaml").write_text(
+ "catalog:\n snapshot:\n type: rest\n",
+ encoding="utf-8",
+ )
+ (process_home / ".pyiceberg.yaml").write_text(
+ "catalog:\n live-process:\n type: hive\n",
+ encoding="utf-8",
+ )
+
+ with (
+ patch.dict(
+ os.environ,
+ {"PYICEBERG_HOME": str(process_home)},
+ clear=False,
+ ),
+ patch(
+ "marimo._data.data_source_discovery.plugins.pyiceberg."
+ "DependencyManager.pyiceberg.has",
+ return_value=True,
+ ) as has_pyiceberg,
+ ):
+ detected = discover(
+ DiscoveryContext(environment={"PYICEBERG_HOME": str(context_home)})
+ )
+
+ assert [source.id for source in detected] == snapshot(
+ ["pyiceberg-snapshot"]
+ )
+ has_pyiceberg.assert_not_called()
+
+
+def test_context_environment_overrides_file_catalog_metadata(
+ tmp_path: Path,
+) -> None:
+ configured_home = tmp_path / "configured"
+ configured_home.mkdir()
+ (configured_home / ".pyiceberg.yaml").write_text(
+ "catalog:\n prod:\n type: rest\n",
+ encoding="utf-8",
+ )
+
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PYICEBERG_HOME": str(configured_home),
+ "PYICEBERG_CATALOG__PROD__TYPE": "glue",
+ }
+ )
+ )
+
+ resolved_types: list[str] = []
+ for configuration in detected[0].configuration:
+ if configuration.field != "Type":
+ continue
+ value = configuration.value
+ assert isinstance(value, SafeLiteralDiscoveryValue)
+ resolved_types.append(value.value)
+
+ assert [source.id for source in detected] == snapshot(["pyiceberg-prod"])
+ assert resolved_types == snapshot(["Glue"])
+
+
+def test_empty_configuration_falls_through_to_next_location(
+ tmp_path: Path,
+) -> None:
+ configured_home = tmp_path / "configured"
+ home_directory = tmp_path / "home"
+ current_directory = tmp_path / "cwd"
+ for directory in (configured_home, home_directory, current_directory):
+ directory.mkdir()
+
+ (configured_home / ".pyiceberg.yaml").write_text(
+ "# intentionally empty\n",
+ encoding="utf-8",
+ )
+ (home_directory / ".pyiceberg.yaml").write_text(
+ "catalog:\n home:\n type: rest\n",
+ encoding="utf-8",
+ )
+
+ with patch(
+ "marimo._data.data_source_discovery.plugins.pyiceberg."
+ "DependencyManager.pyiceberg.has",
+ return_value=False,
+ ):
+ catalogs = _load_resolved_catalogs(
+ environment={"PYICEBERG_HOME": str(configured_home)},
+ home_directory=home_directory,
+ current_directory=current_directory,
+ )
+
+ assert list(catalogs) == snapshot(["home"])
diff --git a/tests/_data/data_source_discovery/test_mysql.py b/tests/_data/data_source_discovery/test_mysql.py
new file mode 100644
index 0000000000..73c4fa162a
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_mysql.py
@@ -0,0 +1,222 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import msgspec
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery.plugins.mysql import discover
+from marimo._data.data_source_discovery.types import DiscoveryContext
+
+
+def test_discovers_mysql_environment() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "MYSQL_HOST": "secret-host",
+ "MYSQL_TCP_PORT": "3306",
+ "MYSQL_USER": "secret-user",
+ "MYSQL_PWD": "secret-password",
+ "MYSQL_DATABASE": "secret-database",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins == snapshot(
+ [
+ {
+ "id": "mysql-environment",
+ "integration": "mysql",
+ "category": "database",
+ "displayName": "MySQL",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "Host",
+ "value": {
+ "kind": "environment-variable",
+ "name": "MYSQL_HOST",
+ },
+ },
+ {
+ "field": "Username",
+ "value": {
+ "kind": "environment-variable",
+ "name": "MYSQL_USER",
+ },
+ },
+ {
+ "field": "Password",
+ "value": {
+ "kind": "environment-variable",
+ "name": "MYSQL_PWD",
+ },
+ },
+ {
+ "field": "Database",
+ "value": {
+ "kind": "environment-variable",
+ "name": "MYSQL_DATABASE",
+ },
+ },
+ {
+ "field": "Port",
+ "value": {
+ "kind": "environment-variable",
+ "name": "MYSQL_TCP_PORT",
+ },
+ },
+ ],
+ "code": """\
+import os
+import sqlalchemy
+
+DATABASE_URL = sqlalchemy.URL.create(
+ "mysql+pymysql",
+ username=os.environ["MYSQL_USER"],
+ password=os.environ["MYSQL_PWD"],
+ host=os.environ["MYSQL_HOST"],
+ port=int(os.environ["MYSQL_TCP_PORT"]),
+ database=os.environ["MYSQL_DATABASE"],
+)
+engine = sqlalchemy.create_engine(DATABASE_URL)""",
+ }
+ ]
+ )
+ assert "secret-" not in repr(builtins)
+
+
+def test_ignores_partial_mysql_environment() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={"MYSQL_HOST": "host", "MYSQL_USER": "user"}
+ )
+ )
+
+ assert detected == snapshot([])
+
+
+def test_discovers_mysql_password_environment() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "MYSQL_HOST": "secret-host",
+ "MYSQL_USER": "secret-user",
+ "MYSQL_PASSWORD": "secret-password",
+ "MYSQL_DATABASE": "secret-database",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins[0]["configuration"][2] == snapshot(
+ {
+ "field": "Password",
+ "value": {
+ "kind": "environment-variable",
+ "name": "MYSQL_PASSWORD",
+ },
+ }
+ )
+ assert 'password=os.environ["MYSQL_PASSWORD"]' in builtins[0]["code"]
+ assert "secret-" not in repr(builtins)
+
+
+def test_prefers_mysql_password_over_mysql_pwd() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "MYSQL_HOST": "host",
+ "MYSQL_USER": "user",
+ "MYSQL_PASSWORD": "password",
+ "MYSQL_PWD": "legacy-password",
+ "MYSQL_DATABASE": "database",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins[0]["configuration"][2] == snapshot(
+ {
+ "field": "Password",
+ "value": {
+ "kind": "environment-variable",
+ "name": "MYSQL_PASSWORD",
+ },
+ }
+ )
+ assert 'password=os.environ["MYSQL_PASSWORD"]' in detected[0].code
+ assert "MYSQL_PWD" not in detected[0].code
+
+
+def test_ignores_empty_required_value() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "MYSQL_HOST": "host",
+ "MYSQL_USER": "user",
+ "MYSQL_PWD": "",
+ "MYSQL_DATABASE": "database",
+ }
+ )
+ )
+
+ assert detected == snapshot([])
+
+
+def test_ignores_environment_without_password() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "MYSQL_HOST": "host",
+ "MYSQL_USER": "user",
+ "MYSQL_DATABASE": "database",
+ }
+ )
+ )
+
+ assert detected == snapshot([])
+
+
+def test_uses_default_for_empty_optional_port() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "MYSQL_HOST": "host",
+ "MYSQL_USER": "user",
+ "MYSQL_PWD": "password",
+ "MYSQL_DATABASE": "database",
+ "MYSQL_TCP_PORT": "",
+ }
+ )
+ )
+
+ assert "port=3306" in detected[0].code
+ assert "MYSQL_TCP_PORT" not in detected[0].code
+
+
+def test_uses_default_for_malformed_optional_port() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "MYSQL_HOST": "host",
+ "MYSQL_USER": "user",
+ "MYSQL_PWD": "password",
+ "MYSQL_DATABASE": "database",
+ "MYSQL_TCP_PORT": "not-a-port",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert [
+ item["field"] for item in builtins[0]["configuration"]
+ ] == snapshot(["Host", "Username", "Password", "Database"])
+ assert "port=3306" in detected[0].code
+ assert "MYSQL_TCP_PORT" not in detected[0].code
diff --git a/tests/_data/data_source_discovery/test_postgres.py b/tests/_data/data_source_discovery/test_postgres.py
new file mode 100644
index 0000000000..6c917061a2
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_postgres.py
@@ -0,0 +1,172 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import msgspec
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery.plugins.postgres import discover
+from marimo._data.data_source_discovery.types import DiscoveryContext
+
+
+def test_discovers_libpq_environment() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PGHOST": "secret-host",
+ "PGPORT": "5432",
+ "PGUSER": "secret-user",
+ "PGPASSWORD": "secret-password",
+ "PGDATABASE": "secret-database",
+ }
+ )
+ )
+
+ assert msgspec.json.decode(msgspec.json.encode(detected)) == snapshot(
+ [
+ {
+ "id": "postgres-libpq-environment",
+ "integration": "postgres",
+ "category": "database",
+ "displayName": "PostgreSQL",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "Host",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGHOST",
+ },
+ },
+ {
+ "field": "Username",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGUSER",
+ },
+ },
+ {
+ "field": "Database",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGDATABASE",
+ },
+ },
+ {
+ "field": "Port",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGPORT",
+ },
+ },
+ {
+ "field": "Password",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGPASSWORD",
+ },
+ },
+ ],
+ "code": """\
+import os
+import sqlalchemy
+
+DATABASE_URL = sqlalchemy.URL.create(
+ "postgresql",
+ username=os.environ["PGUSER"],
+ password=os.environ["PGPASSWORD"],
+ host=os.environ["PGHOST"],
+ port=int(os.environ["PGPORT"]),
+ database=os.environ["PGDATABASE"],
+)
+engine = sqlalchemy.create_engine(DATABASE_URL)""",
+ }
+ ]
+ )
+
+
+def test_ignores_partial_libpq_environment() -> None:
+ detected = discover(
+ DiscoveryContext(environment={"PGHOST": "host", "PGUSER": "user"})
+ )
+
+ assert detected == snapshot([])
+
+
+def test_ignores_empty_required_value() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PGHOST": "host",
+ "PGUSER": "",
+ "PGDATABASE": "database",
+ }
+ )
+ )
+
+ assert detected == snapshot([])
+
+
+def test_uses_defaults_for_empty_optional_values() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PGHOST": "host",
+ "PGUSER": "user",
+ "PGDATABASE": "database",
+ "PGPORT": "",
+ "PGPASSWORD": "",
+ }
+ )
+ )
+
+ assert "port=5432" in detected[0].code
+ assert "PGPORT" not in detected[0].code
+ assert "PGPASSWORD" not in detected[0].code
+
+
+def test_uses_default_for_malformed_optional_port() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "PGHOST": "host",
+ "PGUSER": "user",
+ "PGDATABASE": "database",
+ "PGPORT": "not-a-port",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins[0]["configuration"] == snapshot(
+ [
+ {
+ "field": "Host",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGHOST",
+ },
+ },
+ {
+ "field": "Username",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGUSER",
+ },
+ },
+ {
+ "field": "Database",
+ "value": {
+ "kind": "environment-variable",
+ "name": "PGDATABASE",
+ },
+ },
+ ]
+ )
+ assert "port=5432" in detected[0].code
+ assert "PGPORT" not in detected[0].code
diff --git a/tests/_data/data_source_discovery/test_pyspark.py b/tests/_data/data_source_discovery/test_pyspark.py
new file mode 100644
index 0000000000..3d9154bd0e
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_pyspark.py
@@ -0,0 +1,116 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import ast
+import sys
+from types import ModuleType, SimpleNamespace
+from typing import TYPE_CHECKING
+
+import msgspec
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery.plugins.pyspark import discover
+from marimo._data.data_source_discovery.types import DiscoveryContext
+
+if TYPE_CHECKING:
+ import pytest
+
+
+def test_discovers_spark_connect_environment() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={"SPARK_REMOTE": "sc://secret-host:61235"}
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins == snapshot(
+ [
+ {
+ "id": "pyspark-connect-environment",
+ "integration": "pyspark",
+ "category": "database",
+ "displayName": "PySpark (Spark Connect)",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "Remote",
+ "value": {
+ "kind": "environment-variable",
+ "name": "SPARK_REMOTE",
+ },
+ }
+ ],
+ "code": """\
+import os
+import ibis
+from pyspark.sql import SparkSession
+
+session = SparkSession.builder.remote(
+ os.environ["SPARK_REMOTE"]
+).getOrCreate()
+con = ibis.pyspark.connect(session)""",
+ }
+ ]
+ )
+ ast.parse(detected[0].code)
+ assert "secret-host" not in repr(builtins)
+
+
+def test_ignores_missing_or_empty_spark_remote() -> None:
+ assert discover(DiscoveryContext(environment={})) == snapshot([])
+ assert discover(
+ DiscoveryContext(environment={"SPARK_REMOTE": ""})
+ ) == snapshot([])
+ assert discover(
+ DiscoveryContext(environment={"SPARK_HOME": "/secret/spark"})
+ ) == snapshot([])
+
+
+def test_generated_code_creates_remote_ibis_connection(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ remote_urls: list[str] = []
+ session = object()
+ connections: list[object] = []
+
+ class Builder:
+ def remote(self, url: str) -> Builder:
+ remote_urls.append(url)
+ return self
+
+ def getOrCreate(self) -> object:
+ return session
+
+ class SparkSession:
+ builder = Builder()
+
+ pyspark = ModuleType("pyspark")
+ pyspark_sql = ModuleType("pyspark.sql")
+ pyspark_sql.SparkSession = SparkSession # type: ignore[attr-defined]
+ ibis = ModuleType("ibis")
+ ibis.pyspark = SimpleNamespace( # type: ignore[attr-defined]
+ connect=lambda value: connections.append(value) or "connection"
+ )
+ monkeypatch.setitem(sys.modules, "pyspark", pyspark)
+ monkeypatch.setitem(sys.modules, "pyspark.sql", pyspark_sql)
+ monkeypatch.setitem(sys.modules, "ibis", ibis)
+ monkeypatch.setenv("SPARK_REMOTE", "sc://spark.example:15002")
+
+ source = discover(
+ DiscoveryContext(
+ environment={"SPARK_REMOTE": "sc://spark.example:15002"}
+ )
+ )[0]
+ namespace: dict[str, object] = {}
+ exec(compile(source.code, "<pyspark-discovery>", "exec"), namespace)
+
+ assert remote_urls == snapshot(["sc://spark.example:15002"])
+ assert connections == [session]
+ assert namespace["con"] == "connection"
diff --git a/tests/_data/data_source_discovery/test_trino.py b/tests/_data/data_source_discovery/test_trino.py
new file mode 100644
index 0000000000..ebe65f2351
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_trino.py
@@ -0,0 +1,265 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import ast
+
+import msgspec
+import pytest
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery.plugins.trino import discover
+from marimo._data.data_source_discovery.types import DiscoveryContext
+
+
+def test_discovers_complete_trino_environment() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "TRINO_HOST": "secret-host",
+ "TRINO_PORT": "61234",
+ "TRINO_USER": "secret-user",
+ "TRINO_PASSWORD": "secret-password",
+ "TRINO_CATALOG": "secret-catalog",
+ "TRINO_SCHEMA": "secret-schema",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins == snapshot(
+ [
+ {
+ "id": "trino-environment",
+ "integration": "trino",
+ "category": "database",
+ "displayName": "Trino",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "Host",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_HOST",
+ },
+ },
+ {
+ "field": "Port",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_PORT",
+ },
+ },
+ {
+ "field": "Username",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_USER",
+ },
+ },
+ {
+ "field": "Password",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_PASSWORD",
+ },
+ },
+ {
+ "field": "Catalog",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_CATALOG",
+ },
+ },
+ {
+ "field": "Schema",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_SCHEMA",
+ },
+ },
+ ],
+ "code": """\
+import os
+import sqlalchemy
+import trino.sqlalchemy
+from trino.auth import BasicAuthentication
+
+connect_args = {
+ "auth": BasicAuthentication(
+ os.environ["TRINO_USER"],
+ os.environ["TRINO_PASSWORD"],
+ ),
+ "http_scheme": "https",
+}
+
+database = os.environ["TRINO_CATALOG"]
+database += "/" + os.environ["TRINO_SCHEMA"]
+TRINO_URL = sqlalchemy.URL.create(
+ "trino",
+ username=os.environ["TRINO_USER"],
+ host=os.environ["TRINO_HOST"],
+ port=int(os.environ["TRINO_PORT"]),
+ database=database,
+)
+engine = sqlalchemy.create_engine(TRINO_URL, connect_args=connect_args)""",
+ }
+ ]
+ )
+ ast.parse(detected[0].code)
+ assert "secret-" not in repr(builtins)
+
+
+def test_uses_trino_defaults_for_optional_variables() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "TRINO_HOST": "secret-host",
+ "TRINO_USER": "secret-user",
+ "TRINO_CATALOG": "secret-catalog",
+ "TRINO_PORT": "",
+ "TRINO_PASSWORD": "",
+ "TRINO_SCHEMA": "",
+ }
+ )
+ )
+
+ assert msgspec.json.decode(msgspec.json.encode(detected)) == snapshot(
+ [
+ {
+ "id": "trino-environment",
+ "integration": "trino",
+ "category": "database",
+ "displayName": "Trino",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "Host",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_HOST",
+ },
+ },
+ {
+ "field": "Username",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_USER",
+ },
+ },
+ {
+ "field": "Catalog",
+ "value": {
+ "kind": "environment-variable",
+ "name": "TRINO_CATALOG",
+ },
+ },
+ ],
+ "code": """\
+import os
+import sqlalchemy
+import trino.sqlalchemy
+
+database = os.environ["TRINO_CATALOG"]
+TRINO_URL = sqlalchemy.URL.create(
+ "trino",
+ username=os.environ["TRINO_USER"],
+ host=os.environ["TRINO_HOST"],
+ port=8080,
+ database=database,
+)
+engine = sqlalchemy.create_engine(TRINO_URL)""",
+ }
+ ]
+ )
+
+
+def test_password_without_port_does_not_change_default_port() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "TRINO_HOST": "secret-host",
+ "TRINO_USER": "secret-user",
+ "TRINO_PASSWORD": "secret-password",
+ "TRINO_CATALOG": "secret-catalog",
+ }
+ )
+ )
+
+ assert [
+ line.strip()
+ for line in detected[0].code.splitlines()
+ if "http_scheme" in line or "port=" in line
+ ] == snapshot(
+ [
+ '"http_scheme": "https",',
+ "port=8080,",
+ ]
+ )
+
+
+ "port",
+ [
+ "not-a-port",
+ "0",
+ "-1",
+ "65536",
+ ],
+)
+def test_invalid_port_uses_default(port: str) -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "TRINO_HOST": "secret-host",
+ "TRINO_PORT": port,
+ "TRINO_USER": "secret-user",
+ "TRINO_CATALOG": "secret-catalog",
+ }
+ )
+ )
+
+ assert [
+ configuration.field for configuration in detected[0].configuration
+ ] == snapshot(["Host", "Username", "Catalog"])
+ assert "port=8080" in detected[0].code
+ assert "TRINO_PORT" not in detected[0].code
+
+
+ "environment",
+ [
+ {},
+ {"TRINO_HOST": "host", "TRINO_USER": "user"},
+ {
+ "TRINO_HOST": "",
+ "TRINO_USER": "user",
+ "TRINO_CATALOG": "catalog",
+ },
+ {
+ "TRINO_HOST": "host",
+ "TRINO_USER": "",
+ "TRINO_CATALOG": "catalog",
+ },
+ {
+ "TRINO_HOST": "host",
+ "TRINO_USER": "user",
+ "TRINO_CATALOG": "",
+ },
+ ],
+)
+def test_ignores_incomplete_trino_environment(
+ environment: dict[str, str],
+) -> None:
+ assert discover(DiscoveryContext(environment=environment)) == snapshot([])
diff --git a/tests/_internal/snapshots/internal_api.txt b/tests/_internal/snapshots/internal_api.txt
index e4a9b851cc..aac84ef919 100644
--- a/tests/_internal/snapshots/internal_api.txt
+++ b/tests/_internal/snapshots/internal_api.txt
@@ -7,6 +7,7 @@ commands
CreateNotebookCommand
DebugCellCommand
DeleteCellCommand
+ DiscoverDataSourcesCommand
ExecuteCellCommand
ExecuteCellsCommand
ExecuteScratchpadCommand
diff --git a/tests/_pyodide/test_pyodide_session.py b/tests/_pyodide/test_pyodide_session.py
index 4b6fa27a3b..135fe1895f 100644
--- a/tests/_pyodide/test_pyodide_session.py
+++ b/tests/_pyodide/test_pyodide_session.py
@@ -536,9 +536,11 @@ async def test_pyodide_session_put_input(
[
# Notebook operations
(
- '{"type": "create-notebook", "executionRequests": [{"cellId": "cell-1", "code": "print(1)", "type": "execute-cell"}], '
- '"cellIds": ["cell-1"], "setUiElementValueRequest": {"objectIds": [], "values": [], "type": "update-ui-element"}, '
- '"autoRun": true}',
+ (
+ '{"type": "create-notebook", "executionRequests": [{"cellId": "cell-1", "code": "print(1)", "type": "execute-cell"}], '
+ '"cellIds": ["cell-1"], "setUiElementValueRequest": {"objectIds": [], "values": [], "type": "update-ui-element"}, '
+ '"autoRun": true}'
+ ),
CreateNotebookCommand,
),
(
@@ -605,8 +607,10 @@ async def test_pyodide_session_put_input(
PreviewDatasetColumnCommand,
),
(
- '{"type": "preview-sql-table", "requestId": "req-1", "engine": "duckdb", "database": "test.db", '
- '"schema": "main", "tableName": "users"}',
+ (
+ '{"type": "preview-sql-table", "requestId": "req-1", "engine": "duckdb", "database": "test.db", '
+ '"schema": "main", "tableName": "users"}'
+ ),
PreviewSQLTableCommand,
),
(
diff --git a/tests/_runtime/test_runtime_data_source_discovery.py b/tests/_runtime/test_runtime_data_source_discovery.py
new file mode 100644
index 0000000000..d5281ae236
--- /dev/null
+++ b/tests/_runtime/test_runtime_data_source_discovery.py
@@ -0,0 +1,57 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import os
+from unittest.mock import patch
+
+from marimo._messaging.notification import (
+ DataSourceDiscoveryResultNotification,
+)
+from marimo._runtime.commands import DiscoverDataSourcesCommand
+from marimo._types.ids import RequestId
+from tests.conftest import MockedKernel
+
+
+async def test_discovery_reads_live_kernel_environment(
+ mocked_kernel: MockedKernel,
+) -> None:
+ kernel = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ with (
+ patch.dict(
+ os.environ,
+ {
+ "PGHOST": "host",
+ "PGUSER": "user",
+ "PGDATABASE": "database",
+ },
+ clear=True,
+ ),
+ patch(
+ "marimo._data.data_source_discovery.plugins.pyiceberg."
+ "_load_resolved_catalogs",
+ return_value={},
+ ) as load_resolved_catalogs,
+ ):
+ await kernel.handle_message(
+ DiscoverDataSourcesCommand(request_id=RequestId("discovery"))
+ )
+
+ results = [
+ operation
+ for operation in stream.operations
+ if isinstance(operation, DataSourceDiscoveryResultNotification)
+ ]
+ assert len(results) == 1
+ assert results[0].request_id == RequestId("discovery")
+ assert [source.integration for source in results[0].sources] == [
+ "postgres"
+ ]
+ load_resolved_catalogs.assert_called_once_with(
+ environment={
+ "PGHOST": "host",
+ "PGUSER": "user",
+ "PGDATABASE": "database",
+ }
+ )
diff --git a/tests/_server/api/endpoints/test_datasources.py b/tests/_server/api/endpoints/test_datasources.py
index fb9fb167fb..6587867745 100644
--- a/tests/_server/api/endpoints/test_datasources.py
+++ b/tests/_server/api/endpoints/test_datasources.py
@@ -15,6 +15,17 @@ HEADERS = {
}
+@with_session(SESSION_ID)
+def test_discover_data_sources(client: TestClient) -> None:
+ response = client.post(
+ "/api/datasources/discover",
+ headers=HEADERS,
+ json={"requestId": "discovery-request"},
+ )
+ assert response.status_code == 200, response.text
+ assert response.json()["success"] is True
+
+
@with_session(SESSION_ID)
def test_preview_column(client: TestClient) -> None:
response = client.post(
@@ -99,6 +110,13 @@ def test_preview_datasource_connection(client: TestClient) -> None:
@with_read_session(SESSION_ID)
def test_fails_in_read_mode(client: TestClient) -> None:
+ response = client.post(
+ "/api/datasources/discover",
+ headers=HEADERS,
+ json={"requestId": "discovery-request"},
+ )
+ assert response.status_code == 401
+
response = client.post(
"/api/datasources/preview_column",
headers=HEADERS,
diff --git a/tests/_server/api/endpoints/test_execution.py b/tests/_server/api/endpoints/test_execution.py
index 06f920625d..549f4151e2 100644
--- a/tests/_server/api/endpoints/test_execution.py
+++ b/tests/_server/api/endpoints/test_execution.py
@@ -378,11 +378,14 @@ class TestExecutionRoutes_EditMode:
json={
"cellIds": ["test-1"],
"codes": [
- "import marimo as mo\n"
- "import json\n"
- "request = dict(mo.app_meta().request)\n"
- "request['user'] = bool(request['user'])\n" # user is not serializable
- "print(json.dumps(request))"
+ (
+ "import marimo as mo\n"
+ "import json\n"
+ "request = dict(mo.app_meta().request)\n"
+ # user is not serializable
+ "request['user'] = bool(request['user'])\n"
+ "print(json.dumps(request))"
+ )
],
},
)
@@ -551,11 +554,14 @@ class TestExecutionRoutes_RunMode:
json={
"cellIds": ["test-1"],
"codes": [
- "import marimo as mo\n"
- "import json\n"
- "request = dict(mo.app_meta().request)\n"
- "request['user'] = bool(request['user'])\n" # user is not serializable
- "print(json.dumps(request))"
+ (
+ "import marimo as mo\n"
+ "import json\n"
+ "request = dict(mo.app_meta().request)\n"
+ # user is not serializable
+ "request['user'] = bool(request['user'])\n"
+ "print(json.dumps(request))"
+ )
],
},
)
diff --git a/tests/_utils/test_format_signature.py b/tests/_utils/test_format_signature.py
index 50ab4e530c..bdedb9e88b 100644
--- a/tests/_utils/test_format_signature.py
+++ b/tests/_utils/test_format_signature.py
@@ -21,12 +21,14 @@ class TestFormatSignature:
("func(x: int, y: str = None) -> bool", "bool"),
# Long (like hstack)
(
- "hstack(items: Sequence[object], *, "
- 'justify: str = "space-between", '
- "align: str = None, "
- "wrap: bool = False, "
- "gap: float = 0.5, "
- "widths: str = None) -> Html",
+ (
+ "hstack(items: Sequence[object], *, "
+ 'justify: str = "space-between", '
+ "align: str = None, "
+ "wrap: bool = False, "
+ "gap: float = 0.5, "
+ "widths: str = None) -> Html"
+ ),
"Html",
),
],
diff --git a/tests/snapshots/dependencies.txt b/tests/snapshots/dependencies.txt
index b87d1ffdd0..d3e2678d34 100644
--- a/tests/snapshots/dependencies.txt
+++ b/tests/snapshots/dependencies.txt
@@ -15,6 +15,6 @@ pyyaml>=6.0.1
pyzmq>=27.1.0; python_version < '3.15' and sys_platform != 'emscripten'
starlette>=0.37.2; sys_platform != 'emscripten'
tomlkit>=0.12.0
-typing_extensions>=4.4.0; python_version < '3.11'
+typing_extensions>=4.4.0; python_version < '3.12'
uvicorn>=0.22.0; sys_platform != 'emscripten'
websockets >= 14.2.0; sys_platform != 'emscripten'
\ No newline at end of file