Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

data sources: surface env detections, hide them once connected (#10575)

Shahmir Varqhaba0219a926ed
diff --git a/frontend/src/components/datasources/datasources.tsx b/frontend/src/components/datasources/datasources.tsx
index 54ce7cf663..e9ab277dbb 100644
--- a/frontend/src/components/datasources/datasources.tsx
+++ b/frontend/src/components/datasources/datasources.tsx
@@ -21,15 +21,11 @@ import { cellIdsAtom, useCellActions } from "@/core/cells/cells";
import { useLastFocusedCellId } from "@/core/cells/focus";
import { autoInstantiateAtom } from "@/core/config/config";
import {
- dataConnectionsMapAtom,
+ connectionsAtom,
type SQLTableContext,
useDataSourceActions,
} from "@/core/datasets/data-source-connections";
-import {
- DEFAULT_DUCKDB_DATABASE,
- DUCKDB_ENGINE,
- INTERNAL_SQL_ENGINES,
-} from "@/core/datasets/engines";
+import { DUCKDB_ENGINE } from "@/core/datasets/engines";
import {
PreviewSQLSchemaList,
PreviewSQLTable,
@@ -224,40 +220,6 @@ export function filterEmptyDatabases(databases: Database[]): Database[] {
return changed ? result : databases;
}
-/**
- * This atom is used to get the data connections that are available to the user.
- * It filters out the internal engines if it has no databases or if it has only the in-memory database and no schemas.
- */
-export const connectionsAtom = atom((get) => {
- const dataConnections = new Map(get(dataConnectionsMapAtom));
-
- // Filter out the internal engines if it has no databases
- // Or if it has only the in-memory database and no schemas
- for (const engine of INTERNAL_SQL_ENGINES) {
- const connection = dataConnections.get(engine);
- if (!connection) {
- continue;
- }
-
- if (connection.databases.length === 0) {
- dataConnections.delete(engine);
- }
-
- if (
- connection.databases.length === 1 &&
- connection.databases[0].name === DEFAULT_DUCKDB_DATABASE &&
- connection.databases[0].schemas.length === 0
- ) {
- dataConnections.delete(engine);
- }
- }
-
- // Put internal engines last to prioritize user-defined connections
- return sortBy([...dataConnections.values()], (connection) =>
- INTERNAL_SQL_ENGINES.has(connection.name) ? 1 : 0,
- );
-});
-
export const DataSources: React.FC = () => {
const [searchValue, setSearchValue] = React.useState<string>("");
const [hideEmpty, setHideEmpty] = useAtom(hideEmptyDatasourcesAtom);
diff --git a/frontend/src/components/editor/chrome/panels/components.tsx b/frontend/src/components/editor/chrome/panels/components.tsx
index 548e74377d..0a973a469f 100644
--- a/frontend/src/components/editor/chrome/panels/components.tsx
+++ b/frontend/src/components/editor/chrome/panels/components.tsx
@@ -8,6 +8,9 @@ import {
} from "@/components/ui/accordion";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/utils/cn";
+import { SparklesIcon } from "lucide-react";
+import { Tooltip } from "@/components/ui/tooltip";
+import type { DataSourceDiscoveryGroup } from "@/core/datasets/data-source-discovery";
/**
* Styled AccordionItem for sidebar panels.
@@ -80,9 +83,41 @@ const PanelBadge = ({
);
PanelBadge.displayName = "PanelBadge";
+const DiscoveredSourcesBadge = ({
+ count,
+ type,
+}: {
+ count: number;
+ type: DataSourceDiscoveryGroup;
+}) => {
+ if (count === 0) {
+ return null;
+ }
+
+ let noun: string;
+ if (type === "database") {
+ noun = count === 1 ? "database" : "databases";
+ } else {
+ noun = count === 1 ? "remote storage" : "remote storages";
+ }
+ const content = `${count} ${noun} detected in your environment, ready to quick-add`;
+
+ return (
+ <Tooltip content={content}>
+ <span>
+ <PanelBadge className="gap-0.5 border-(--blue-6) bg-(--blue-3) text-(--blue-11)">
+ <SparklesIcon className="h-2.5 w-2.5" />
+ {count}
+ </PanelBadge>
+ </span>
+ </Tooltip>
+ );
+};
+
export {
PanelAccordionItem,
PanelAccordionTrigger,
PanelAccordionContent,
PanelBadge,
+ DiscoveredSourcesBadge,
};
diff --git a/frontend/src/components/editor/chrome/panels/file-explorer-panel.tsx b/frontend/src/components/editor/chrome/panels/file-explorer-panel.tsx
index 96a974edd2..50e2362a9c 100644
--- a/frontend/src/components/editor/chrome/panels/file-explorer-panel.tsx
+++ b/frontend/src/components/editor/chrome/panels/file-explorer-panel.tsx
@@ -7,11 +7,13 @@ import useResizeObserver from "use-resize-observer";
import { StorageInspector } from "@/components/storage/storage-inspector";
import { Accordion } from "@/components/ui/accordion";
import { storageNamespacesAtom } from "@/core/storage/state";
+import { useDetectedDataSources } from "@/hooks/useDataSourceDiscovery";
import { cn } from "@/utils/cn";
import { TreeDndProvider } from "../../file-tree/dnd-wrapper";
import { FileExplorer } from "../../file-tree/file-explorer";
import { useFileExplorerUpload } from "../../file-tree/upload";
import {
+ DiscoveredSourcesBadge,
PanelAccordionContent,
PanelAccordionItem,
PanelAccordionTrigger,
@@ -58,6 +60,7 @@ const FileExplorerPanel: React.FC = () => {
const storageNamespaces = useAtomValue(storageNamespacesAtom);
const remoteStorageConnections = storageNamespaces.length;
+ const pendingDataSources = useDetectedDataSources("storage");
const openSections = useMemo<FileExplorerPanelSection[]>(() => {
if (!state.hasUserInteracted && remoteStorageConnections > 0) {
@@ -81,6 +84,7 @@ const FileExplorerPanel: React.FC = () => {
const availableContent = panelHeight - TRIGGER_HEIGHT * 2;
const storageIsOpen = openSections.includes("remote-storage");
+ const showDiscoveredStorageBadge = pendingDataSources.length > 0;
const bothOpen = storageIsOpen && openSections.includes("files");
const storageMaxHeight = bothOpen
@@ -104,6 +108,12 @@ const FileExplorerPanel: React.FC = () => {
{remoteStorageConnections > 0 && (
<PanelBadge>{remoteStorageConnections}</PanelBadge>
)}
+ {showDiscoveredStorageBadge && (
+ <DiscoveredSourcesBadge
+ count={pendingDataSources.length}
+ type="storage"
+ />
+ )}
</PanelAccordionTrigger>
<PanelAccordionContent
className="overflow-auto"
diff --git a/frontend/src/components/editor/chrome/panels/session-panel.tsx b/frontend/src/components/editor/chrome/panels/session-panel.tsx
index c42d5a4a6a..778c1e0a3a 100644
--- a/frontend/src/components/editor/chrome/panels/session-panel.tsx
+++ b/frontend/src/components/editor/chrome/panels/session-panel.tsx
@@ -3,16 +3,16 @@
import { useAtom, useAtomValue } from "jotai";
import { DatabaseIcon, VariableIcon } from "lucide-react";
import React, { useCallback } from "react";
-import {
- connectionsAtom,
- DataSources,
-} from "@/components/datasources/datasources";
+import { DataSources } from "@/components/datasources/datasources";
import { Accordion } from "@/components/ui/accordion";
import { VariableTable } from "@/components/variables/variables-table";
import { useCellIds } from "@/core/cells/cells";
+import { connectionsAtom } from "@/core/datasets/data-source-connections";
import { datasetTablesAtom } from "@/core/datasets/state";
import { useVariables } from "@/core/variables/state";
+import { useDetectedDataSources } from "@/hooks/useDataSourceDiscovery";
import {
+ DiscoveredSourcesBadge,
PanelAccordionContent,
PanelAccordionItem,
PanelAccordionTrigger,
@@ -28,6 +28,7 @@ const SessionPanel: React.FC = () => {
const cellIds = useCellIds();
const tables = useAtomValue(datasetTablesAtom);
const dataConnections = useAtomValue(connectionsAtom);
+ const pendingDataSources = useDetectedDataSources("database");
const [state, setState] = useAtom(sessionPanelAtom);
const datasourcesCount = tables.length + dataConnections.length;
@@ -50,6 +51,7 @@ const SessionPanel: React.FC = () => {
const isDatasourcesOpen = openSections.includes("datasources");
const showDatasourcesBadge = !isDatasourcesOpen && datasourcesCount > 0;
+ const showDiscoveredDataSourcesBadge = pendingDataSources.length > 0;
return (
<Accordion
@@ -63,6 +65,12 @@ const SessionPanel: React.FC = () => {
<DatabaseIcon className="w-4 h-4" />
Data sources
{showDatasourcesBadge && <PanelBadge>{datasourcesCount}</PanelBadge>}
+ {showDiscoveredDataSourcesBadge && (
+ <DiscoveredSourcesBadge
+ count={pendingDataSources.length}
+ type="database"
+ />
+ )}
</PanelAccordionTrigger>
<PanelAccordionContent>
<DataSources />
diff --git a/frontend/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx b/frontend/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx
index 3d21d621a8..30520139f1 100644
--- a/frontend/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx
+++ b/frontend/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx
@@ -28,6 +28,7 @@ const sources: DetectedDataSource[] = [
},
],
code: "engine = create_engine()",
+ hidesWhen: { kind: "dialect", substrings: ["postgres"] },
},
{
id: "pyiceberg-prod",
@@ -52,6 +53,7 @@ const sources: DetectedDataSource[] = [
},
],
code: 'catalog = load_catalog("prod")',
+ hidesWhen: { kind: "dialect", substrings: ["iceberg"] },
},
];
diff --git a/frontend/src/components/editor/connections/add-connection-dialog.tsx b/frontend/src/components/editor/connections/add-connection-dialog.tsx
index f5c1d9365b..aa974d8a7b 100644
--- a/frontend/src/components/editor/connections/add-connection-dialog.tsx
+++ b/frontend/src/components/editor/connections/add-connection-dialog.tsx
@@ -87,7 +87,11 @@ export const AddConnectionDialogContent: React.FC<{
<span className="block">{codeSnippetHint}</span>
</DialogDescription>
</DialogHeader>
- <AutoDiscoveredDataSources onSubmit={onClose} className="-mt-2" />
+ <AutoDiscoveredDataSources
+ onSubmit={onClose}
+ className="-mt-2"
+ group={activeTab === "storage" ? "storage" : "database"}
+ />
<Tabs
value={activeTab}
onValueChange={(v) => setActiveTab(v as ConnectionTab)}
diff --git a/frontend/src/components/editor/connections/quick-add-data-sources.tsx b/frontend/src/components/editor/connections/quick-add-data-sources.tsx
index 8f23bc4d12..5ee4ca2fba 100644
--- a/frontend/src/components/editor/connections/quick-add-data-sources.tsx
+++ b/frontend/src/components/editor/connections/quick-add-data-sources.tsx
@@ -2,8 +2,11 @@
import { PlusIcon, SparklesIcon } from "lucide-react";
import { Tooltip, TooltipProvider } from "@/components/ui/tooltip";
-import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
-import { useDataSourceDiscovery } from "@/hooks/useDataSourceDiscovery";
+import type {
+ DataSourceDiscoveryGroup,
+ DetectedDataSource,
+} from "@/core/datasets/data-source-discovery";
+import { useDetectedDataSources } from "@/hooks/useDataSourceDiscovery";
import { useInsertCode } from "./components";
import { cn } from "@/utils/cn";
@@ -94,19 +97,20 @@ const DetectedDataSourceDetails: React.FC<{
);
export const AutoDiscoveredDataSources: React.FC<{
- onSubmit: () => void;
+ onSubmit?: () => void;
className?: string;
-}> = ({ onSubmit, className }) => {
+ group?: DataSourceDiscoveryGroup;
+}> = ({ onSubmit, className, group }) => {
const insertCode = useInsertCode();
- const { data } = useDataSourceDiscovery();
+ const sources = useDetectedDataSources(group);
return (
<QuickAddDataSources
className={className}
- sources={data ?? []}
+ sources={sources}
onAdd={(source) => {
insertCode(source.code);
- onSubmit();
+ onSubmit?.();
}}
/>
);
diff --git a/frontend/src/core/datasets/__tests__/data-source-discovery.test.ts b/frontend/src/core/datasets/__tests__/data-source-discovery.test.ts
new file mode 100644
index 0000000000..109074542b
--- /dev/null
+++ b/frontend/src/core/datasets/__tests__/data-source-discovery.test.ts
@@ -0,0 +1,277 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { connectionAtom } from "@/core/network/connection";
+import { store } from "@/core/state/jotai";
+import { WebSocketState } from "@/core/websocket/types";
+import {
+ fetchDataSourceDiscovery,
+ invalidateDataSourceDiscovery,
+ isDetectedSourceConnected,
+ matchesDiscoveryGroup,
+ type DetectedDataSource,
+} from "../data-source-discovery";
+import { DiscoverDataSources } from "../request-registry";
+
+function source(
+ overrides: Partial<DetectedDataSource> &
+ Pick<DetectedDataSource, "id" | "integration" | "category" | "displayName">,
+): DetectedDataSource {
+ return {
+ confidence: "high",
+ origins: [],
+ configuration: [],
+ code: "",
+ hidesWhen: { kind: "dialect", substrings: [overrides.integration] },
+ ...overrides,
+ };
+}
+
+describe("fetchDataSourceDiscovery", () => {
+ beforeEach(() => {
+ // Discovery waits for the session's websocket connection to be open
+ // before requesting, so it doesn't race a still-connecting session.
+ store.set(connectionAtom, { state: WebSocketState.OPEN });
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ invalidateDataSourceDiscovery();
+ });
+
+ it("shares a single in-flight request across concurrent callers", async () => {
+ const request = vi
+ .spyOn(DiscoverDataSources, "request")
+ .mockResolvedValue({ request_id: "request-id", sources: [] });
+
+ await Promise.all([
+ fetchDataSourceDiscovery(),
+ fetchDataSourceDiscovery(),
+ fetchDataSourceDiscovery(),
+ ]);
+
+ expect(request).toHaveBeenCalledTimes(1);
+ });
+
+ it("re-runs discovery after the cache is invalidated", async () => {
+ const request = vi
+ .spyOn(DiscoverDataSources, "request")
+ .mockResolvedValue({ request_id: "request-id", sources: [] });
+
+ await fetchDataSourceDiscovery();
+ invalidateDataSourceDiscovery();
+ await fetchDataSourceDiscovery();
+
+ expect(request).toHaveBeenCalledTimes(2);
+ });
+
+ it("clears the cache on failure so the next call retries", async () => {
+ const request = vi
+ .spyOn(DiscoverDataSources, "request")
+ .mockRejectedValueOnce(new Error("boom"))
+ .mockResolvedValueOnce({ request_id: "request-id", sources: [] });
+
+ await expect(fetchDataSourceDiscovery()).rejects.toThrow("boom");
+ await expect(fetchDataSourceDiscovery()).resolves.toEqual([]);
+ expect(request).toHaveBeenCalledTimes(2);
+ });
+
+ it("does not clear a newer cache entry when an invalidated request fails", async () => {
+ let rejectFirstRequest: ((error: Error) => void) | undefined;
+ const firstRequest = new Promise<{
+ request_id: string;
+ sources: DetectedDataSource[];
+ }>((_resolve, reject) => {
+ rejectFirstRequest = reject;
+ });
+ const request = vi
+ .spyOn(DiscoverDataSources, "request")
+ .mockReturnValueOnce(firstRequest)
+ .mockResolvedValueOnce({ request_id: "request-id", sources: [] });
+
+ const staleResult = fetchDataSourceDiscovery().catch((error) => error);
+ await vi.waitFor(() => expect(request).toHaveBeenCalledTimes(1));
+ invalidateDataSourceDiscovery();
+ await fetchDataSourceDiscovery();
+
+ rejectFirstRequest?.(new Error("stale request failed"));
+ await staleResult;
+ await fetchDataSourceDiscovery();
+
+ expect(request).toHaveBeenCalledTimes(2);
+ });
+
+ it("returns sources from the kernel", async () => {
+ const sources: DetectedDataSource[] = [
+ source({
+ id: "postgres",
+ integration: "postgres",
+ category: "database",
+ displayName: "PostgreSQL",
+ }),
+ source({
+ id: "mysql",
+ integration: "mysql",
+ category: "database",
+ displayName: "MySQL",
+ }),
+ ];
+ vi.spyOn(DiscoverDataSources, "request").mockResolvedValue({
+ request_id: "request-id",
+ sources,
+ });
+
+ await expect(fetchDataSourceDiscovery()).resolves.toEqual(sources);
+ });
+});
+
+describe("matchesDiscoveryGroup", () => {
+ const postgres = source({
+ id: "postgres",
+ integration: "postgres",
+ category: "database",
+ displayName: "PostgreSQL",
+ });
+ const iceberg = source({
+ id: "iceberg",
+ integration: "pyiceberg",
+ category: "catalog",
+ displayName: "PyIceberg",
+ });
+ const s3 = source({
+ id: "s3",
+ integration: "aws",
+ category: "object-storage",
+ displayName: "S3",
+ hidesWhen: {
+ kind: "storage",
+ protocols: ["s3"],
+ backendTypes: [],
+ },
+ });
+
+ it("keeps every source when no group is given", () => {
+ expect(matchesDiscoveryGroup(postgres)).toBe(true);
+ expect(matchesDiscoveryGroup(iceberg)).toBe(true);
+ expect(matchesDiscoveryGroup(s3)).toBe(true);
+ });
+
+ it("groups catalogs with databases", () => {
+ expect(matchesDiscoveryGroup(postgres, "database")).toBe(true);
+ expect(matchesDiscoveryGroup(iceberg, "database")).toBe(true);
+ expect(matchesDiscoveryGroup(s3, "database")).toBe(false);
+ });
+
+ it("narrows storage to object-storage sources", () => {
+ expect(matchesDiscoveryGroup(postgres, "storage")).toBe(false);
+ expect(matchesDiscoveryGroup(iceberg, "storage")).toBe(false);
+ expect(matchesDiscoveryGroup(s3, "storage")).toBe(true);
+ });
+});
+
+describe("isDetectedSourceConnected", () => {
+ const postgres = source({
+ id: "postgres",
+ integration: "postgres",
+ category: "database",
+ displayName: "PostgreSQL",
+ hidesWhen: { kind: "dialect", substrings: ["postgres"] },
+ });
+ const emptySnapshot = {
+ dialects: [] as string[],
+ storageProtocols: [] as string[],
+ storageBackendTypes: [] as string[],
+ };
+
+ it("does not hide a source when nothing is connected", () => {
+ expect(isDetectedSourceConnected(postgres, emptySnapshot)).toBe(false);
+ });
+
+ it("hides a database suggestion when a matching dialect is live", () => {
+ expect(
+ isDetectedSourceConnected(postgres, {
+ ...emptySnapshot,
+ dialects: ["postgresql"],
+ }),
+ ).toBe(true);
+ });
+
+ it("does not hide a database suggestion for a different dialect", () => {
+ expect(
+ isDetectedSourceConnected(postgres, {
+ ...emptySnapshot,
+ dialects: ["mysql", "duckdb"],
+ }),
+ ).toBe(false);
+ });
+
+ it("hides spark suggestions for either pyspark or spark dialects", () => {
+ const spark = source({
+ id: "spark",
+ integration: "pyspark",
+ category: "catalog",
+ displayName: "Spark",
+ hidesWhen: { kind: "dialect", substrings: ["pyspark", "spark"] },
+ });
+ expect(
+ isDetectedSourceConnected(spark, {
+ ...emptySnapshot,
+ dialects: ["spark"],
+ }),
+ ).toBe(true);
+ });
+
+ it("hides object storage when a matching protocol is live", () => {
+ const s3 = source({
+ id: "s3",
+ integration: "aws",
+ category: "object-storage",
+ displayName: "S3",
+ hidesWhen: {
+ kind: "storage",
+ protocols: ["s3", "cloudflare", "coreweave"],
+ backendTypes: [],
+ },
+ });
+ expect(
+ isDetectedSourceConnected(s3, {
+ ...emptySnapshot,
+ storageProtocols: ["s3"],
+ }),
+ ).toBe(true);
+ });
+
+ it("hides huggingface when the backend type matches", () => {
+ const hf = source({
+ id: "hf",
+ integration: "huggingface",
+ category: "object-storage",
+ displayName: "Hugging Face",
+ hidesWhen: {
+ kind: "storage",
+ protocols: ["hf"],
+ backendTypes: ["huggingface"],
+ },
+ });
+ expect(
+ isDetectedSourceConnected(hf, {
+ ...emptySnapshot,
+ storageBackendTypes: ["huggingface"],
+ }),
+ ).toBe(true);
+ });
+
+ it("does not hide when hidesWhen does not match live connections", () => {
+ expect(
+ isDetectedSourceConnected(
+ source({
+ id: "custom",
+ integration: "custom",
+ category: "database",
+ displayName: "Custom",
+ hidesWhen: { kind: "dialect", substrings: ["custom"] },
+ }),
+ { ...emptySnapshot, dialects: ["postgres"] },
+ ),
+ ).toBe(false);
+ });
+});
diff --git a/frontend/src/core/datasets/data-source-connections.ts b/frontend/src/core/datasets/data-source-connections.ts
index 0f8e0bc8ae..3c8d9beed5 100644
--- a/frontend/src/core/datasets/data-source-connections.ts
+++ b/frontend/src/core/datasets/data-source-connections.ts
@@ -4,6 +4,7 @@ import { atom } from "jotai";
import { isSchemaless } from "@/components/datasources/utils";
import { createReducerAndAtoms } from "@/utils/createReducer";
import { Logger } from "@/utils/Logger";
+import { sortBy } from "@/utils/arrays";
import type {
DatabaseSchema,
DataSourceConnection as DataSourceConnectionType,
@@ -13,6 +14,7 @@ import { store } from "../state/jotai";
import type { VariableName } from "../variables/types";
import {
type ConnectionName,
+ DEFAULT_DUCKDB_DATABASE,
DUCKDB_ENGINE,
INTERNAL_SQL_ENGINES,
} from "./engines";
@@ -341,6 +343,40 @@ export const dataConnectionsMapAtom = atom(
(get) => get(dataSourceConnectionsAtom).connectionsMap,
);
+/**
+ * Get the data connections that are available to the user.
+ * Filters out internal engines if it has no databases or if it has only the in-memory database and no schemas.
+ */
+export const connectionsAtom = atom((get) => {
+ const dataConnections = new Map(get(dataConnectionsMapAtom));
+
+ // Filter out the internal engines if it has no databases
+ // Or if it has only the in-memory database and no schemas
+ for (const engine of INTERNAL_SQL_ENGINES) {
+ const connection = dataConnections.get(engine);
+ if (!connection) {
+ continue;
+ }
+
+ if (connection.databases.length === 0) {
+ dataConnections.delete(engine);
+ }
+
+ if (
+ connection.databases.length === 1 &&
+ connection.databases[0].name === DEFAULT_DUCKDB_DATABASE &&
+ connection.databases[0].schemas.length === 0
+ ) {
+ dataConnections.delete(engine);
+ }
+ }
+
+ // Put internal engines last to prioritize user-defined connections
+ return sortBy([...dataConnections.values()], (connection) =>
+ INTERNAL_SQL_ENGINES.has(connection.name) ? 1 : 0,
+ );
+});
+
export function setLatestEngineSelected(engine: ConnectionName) {
const existing = store.get(dataSourceConnectionsAtom);
// Don't update the map if the engine is not in the map
diff --git a/frontend/src/core/datasets/data-source-discovery.ts b/frontend/src/core/datasets/data-source-discovery.ts
index f5931322a1..f4588fe1bd 100644
--- a/frontend/src/core/datasets/data-source-discovery.ts
+++ b/frontend/src/core/datasets/data-source-discovery.ts
@@ -1,5 +1,97 @@
/* Copyright 2026 Marimo. All rights reserved. */
+import { atom } from "jotai";
import type { components } from "@marimo-team/marimo-api";
+import { waitForConnectionOpenIfNotebook } from "../network/connection";
+import { store } from "../state/jotai";
+import { DiscoverDataSources } from "./request-registry";
export type DetectedDataSource = components["schemas"]["DetectedDataSource"];
+
+export type DataSourceDiscoveryGroup = "database" | "storage";
+
+/** Category filter for UI surfaces. Catalogs ride with databases. */
+export function matchesDiscoveryGroup(
+ source: DetectedDataSource,
+ group?: DataSourceDiscoveryGroup,
+): boolean {
+ if (!group) {
+ return true;
+ }
+ const isStorage = source.category === "object-storage";
+ return isStorage === (group === "storage");
+}
+
+/**
+ * Live engines and storage used to hide suggestions of the same type.
+ */
+export interface LiveConnectionSnapshot {
+ dialects: readonly string[];
+ storageProtocols: readonly string[];
+ storageBackendTypes: readonly string[];
+}
+
+/**
+ * Whether a detected source already has a live connection of the same type.
+ * Match rules come from the kernel on each suggestion.
+ */
+export function isDetectedSourceConnected(
+ source: DetectedDataSource,
+ snapshot: LiveConnectionSnapshot,
+): boolean {
+ const match = source.hidesWhen;
+ if (match.kind === "dialect") {
+ return match.substrings.some((alias) =>
+ snapshot.dialects.some((dialect) => dialect.includes(alias)),
+ );
+ }
+
+ if (
+ match.protocols.some((protocol) =>
+ snapshot.storageProtocols.includes(protocol),
+ )
+ ) {
+ return true;
+ }
+
+ return match.backendTypes.some((backendType) =>
+ snapshot.storageBackendTypes.includes(backendType),
+ );
+}
+
+/**
+ * Bumped when discovery should be re-fetched (kernel restart).
+ */
+export const dataSourceDiscoveryEpochAtom = atom(0);
+
+/**
+ * Kernel-managed datasource discovery scans the environment once per epoch.
+ * Every consumer shares a single in-flight request.
+ */
+let discoveryPromise: Promise<DetectedDataSource[]> | undefined;
+let discoveryEpoch = 0;
+
+export function fetchDataSourceDiscovery(): Promise<DetectedDataSource[]> {
+ const epoch = store.get(dataSourceDiscoveryEpochAtom);
+ if (!discoveryPromise || discoveryEpoch !== epoch) {
+ discoveryEpoch = epoch;
+ discoveryPromise = waitForConnectionOpenIfNotebook()
+ .then(() => DiscoverDataSources.request({}))
+ .then((result) => result.sources)
+ .catch((error: unknown) => {
+ if (discoveryEpoch === epoch) {
+ discoveryPromise = undefined;
+ }
+ throw error;
+ });
+ }
+ return discoveryPromise;
+}
+
+/**
+ * Clears the shared discovery cache and triggers consumers to refetch.
+ */
+export function invalidateDataSourceDiscovery(): void {
+ discoveryPromise = undefined;
+ store.set(dataSourceDiscoveryEpochAtom, (value) => value + 1);
+}
diff --git a/frontend/src/core/websocket/useMarimoKernelConnection.tsx b/frontend/src/core/websocket/useMarimoKernelConnection.tsx
index 261d998abb..ceef54a64a 100644
--- a/frontend/src/core/websocket/useMarimoKernelConnection.tsx
+++ b/frontend/src/core/websocket/useMarimoKernelConnection.tsx
@@ -40,6 +40,7 @@ import type { CellData } from "../cells/types";
import { capabilitiesAtom } from "../config/capabilities";
import { connectionTransportTypeAtom, useSetAppConfig } from "../config/config";
import { useDataSourceActions } from "../datasets/data-source-connections";
+import { invalidateDataSourceDiscovery } from "../datasets/data-source-discovery";
import type { ConnectionName } from "../datasets/engines";
import {
DiscoverDataSources,
@@ -251,6 +252,12 @@ export function useMarimoKernelConnection(opts: {
onError: showBoundary,
existingCells,
});
+ if (!msg.data.resumed) {
+ // A freshly started kernel may expose a different environment
+ // (e.g. new env vars); re-run discovery instead of reusing stale
+ // suggestions from the previous kernel instance.
+ invalidateDataSourceDiscovery();
+ }
setKioskMode(msg.data.kiosk);
// A freshly started kernel has no breakpoints of its own; re-push
// the client's set so they still apply, and clear the stale
@@ -311,6 +318,7 @@ export function useMarimoKernelConnection(opts: {
}
case "variables":
+ const variableNames = msg.data.variables.map((v) => v.name);
setVariables(
msg.data.variables.map((v) => ({
name: v.name,
@@ -318,9 +326,9 @@ export function useMarimoKernelConnection(opts: {
usedBy: v.used_by,
})),
);
- filterDatasetsFromVariables(msg.data.variables.map((v) => v.name));
- filterDataSourcesFromVariables(msg.data.variables.map((v) => v.name));
- filterStorageFromVariables(msg.data.variables.map((v) => v.name));
+ filterDatasetsFromVariables(variableNames);
+ filterDataSourcesFromVariables(variableNames);
+ filterStorageFromVariables(variableNames);
return;
case "variable-values":
setMetadata(
diff --git a/frontend/src/hooks/__tests__/useDataSourceDiscovery.test.ts b/frontend/src/hooks/__tests__/useDataSourceDiscovery.test.ts
deleted file mode 100644
index 42a0bfb8b3..0000000000
--- a/frontend/src/hooks/__tests__/useDataSourceDiscovery.test.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-/* Copyright 2026 Marimo. All rights reserved. */
-
-import { afterEach, describe, expect, it, vi } from "vitest";
-import { DiscoverDataSources } from "@/core/datasets/request-registry";
-import { loadDataSourceDiscovery } from "../useDataSourceDiscovery";
-
-describe("loadDataSourceDiscovery", () => {
- afterEach(() => {
- vi.restoreAllMocks();
- });
-
- it("returns the kernel's secret-free discovery model", async () => {
- const sources = [
- {
- id: "pyiceberg-prod",
- integration: "pyiceberg",
- category: "catalog" as const,
- displayName: "PyIceberg (prod)",
- confidence: "high" as const,
- origins: [
- {
- type: "configuration" as const,
- label: "Resolved PyIceberg configuration",
- },
- ],
- configuration: [
- {
- field: "Catalog",
- value: {
- kind: "safe-literal" as const,
- value: "prod",
- },
- },
- ],
- code: 'catalog = load_catalog("prod")',
- },
- ];
- const request = vi.spyOn(DiscoverDataSources, "request").mockResolvedValue({
- request_id: "request-id",
- sources,
- });
-
- const detected = await loadDataSourceDiscovery();
-
- expect({ detected, requests: request.mock.calls }).toMatchInlineSnapshot(`
- {
- "detected": [
- {
- "category": "catalog",
- "code": "catalog = load_catalog("prod")",
- "confidence": "high",
- "configuration": [
- {
- "field": "Catalog",
- "value": {
- "kind": "safe-literal",
- "value": "prod",
- },
- },
- ],
- "displayName": "PyIceberg (prod)",
- "id": "pyiceberg-prod",
- "integration": "pyiceberg",
- "origins": [
- {
- "label": "Resolved PyIceberg configuration",
- "type": "configuration",
- },
- ],
- },
- ],
- "requests": [
- [
- {},
- ],
- ],
- }
- `);
- });
-});
diff --git a/frontend/src/hooks/useDataSourceDiscovery.ts b/frontend/src/hooks/useDataSourceDiscovery.ts
index fcf41f2d62..48b650e91c 100644
--- a/frontend/src/hooks/useDataSourceDiscovery.ts
+++ b/frontend/src/hooks/useDataSourceDiscovery.ts
@@ -1,18 +1,59 @@
/* Copyright 2026 Marimo. All rights reserved. */
-import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
-import { DiscoverDataSources } from "@/core/datasets/request-registry";
+import { useAtomValue } from "jotai";
+import { useMemo } from "react";
+import { dataConnectionsMapAtom } from "@/core/datasets/data-source-connections";
+import {
+ dataSourceDiscoveryEpochAtom,
+ type DataSourceDiscoveryGroup,
+ type DetectedDataSource,
+ fetchDataSourceDiscovery,
+ isDetectedSourceConnected,
+ matchesDiscoveryGroup,
+} from "@/core/datasets/data-source-discovery";
+import { storageNamespacesAtom } from "@/core/storage/state";
import { useAsyncData } from "./useAsyncData";
-export async function loadDataSourceDiscovery(): Promise<DetectedDataSource[]> {
- const result = await DiscoverDataSources.request({});
- return result.sources;
-}
-
/**
* Reusable UI-facing hook for kernel-managed datasource discovery.
- * Consumers decide how to render, filter, or act on the detected model.
*/
export function useDataSourceDiscovery() {
- return useAsyncData(loadDataSourceDiscovery, []);
+ const epoch = useAtomValue(dataSourceDiscoveryEpochAtom);
+ return useAsyncData(fetchDataSourceDiscovery, [epoch]);
+}
+
+/**
+ * Detected sources from the kernel environment that are not already backed
+ * by a live connection, optionally narrowed to "database" or "storage".
+ */
+export function useDetectedDataSources(
+ group?: DataSourceDiscoveryGroup,
+): DetectedDataSource[] {
+ const { data } = useDataSourceDiscovery();
+ const connections = useAtomValue(dataConnectionsMapAtom);
+ const storageNamespaces = useAtomValue(storageNamespacesAtom);
+
+ return useMemo(() => {
+ if (!data) {
+ return [];
+ }
+
+ const snapshot = {
+ dialects: [...connections.values()].map((connection) =>
+ connection.dialect.toLowerCase(),
+ ),
+ storageProtocols: storageNamespaces.map((namespace) =>
+ namespace.protocol.toLowerCase(),
+ ),
+ storageBackendTypes: storageNamespaces.map((namespace) =>
+ namespace.backendType.toLowerCase(),
+ ),
+ };
+
+ return data.filter(
+ (source) =>
+ matchesDiscoveryGroup(source, group) &&
+ !isDetectedSourceConnected(source, snapshot),
+ );
+ }, [connections, data, group, storageNamespaces]);
}
diff --git a/marimo/_data/data_source_discovery/__init__.py b/marimo/_data/data_source_discovery/__init__.py
index c2af63a4ac..181e489d6c 100644
--- a/marimo/_data/data_source_discovery/__init__.py
+++ b/marimo/_data/data_source_discovery/__init__.py
@@ -4,15 +4,19 @@ from marimo._data.data_source_discovery.models import (
DetectedDataSource,
DetectedDataSourceConfiguration,
DetectedDataSourceOrigin,
+ DialectHidesWhen,
EnvironmentVariableDiscoveryValue,
SafeLiteralDiscoveryValue,
+ StorageHidesWhen,
)
__all__ = [
"DetectedDataSource",
"DetectedDataSourceConfiguration",
"DetectedDataSourceOrigin",
+ "DialectHidesWhen",
"EnvironmentVariableDiscoveryValue",
"SafeLiteralDiscoveryValue",
+ "StorageHidesWhen",
"discover_data_sources",
]
diff --git a/marimo/_data/data_source_discovery/helpers.py b/marimo/_data/data_source_discovery/helpers.py
index 01ff5a1547..b2695bfeb0 100644
--- a/marimo/_data/data_source_discovery/helpers.py
+++ b/marimo/_data/data_source_discovery/helpers.py
@@ -6,8 +6,10 @@ from typing import TYPE_CHECKING
from marimo._data.data_source_discovery.models import (
DetectedDataSourceConfiguration,
DetectedDataSourceOrigin,
+ DialectHidesWhen,
EnvironmentVariableDiscoveryValue,
SafeLiteralDiscoveryValue,
+ StorageHidesWhen,
)
if TYPE_CHECKING:
@@ -57,3 +59,18 @@ def safe_literal(field: str, value: str) -> DetectedDataSourceConfiguration:
field=field,
value=SafeLiteralDiscoveryValue(value=value),
)
+
+
+def hides_when_dialect(*substrings: str) -> DialectHidesWhen:
+ return DialectHidesWhen(substrings=substrings)
+
+
+def hides_when_storage(
+ *,
+ protocols: tuple[str, ...] = (),
+ backend_types: tuple[str, ...] = (),
+) -> StorageHidesWhen:
+ return StorageHidesWhen(
+ protocols=protocols,
+ backend_types=backend_types,
+ )
diff --git a/marimo/_data/data_source_discovery/models.py b/marimo/_data/data_source_discovery/models.py
index 8a1ba7129e..2262317287 100644
--- a/marimo/_data/data_source_discovery/models.py
+++ b/marimo/_data/data_source_discovery/models.py
@@ -51,6 +51,34 @@ class DetectedDataSourceOrigin(msgspec.Struct, frozen=True, rename="camel"):
label: str
+class DialectHidesWhen(
+ msgspec.Struct,
+ frozen=True,
+ tag="dialect",
+ tag_field="kind",
+ rename="camel",
+):
+ """Hide this suggestion when a live SQL engine dialect contains a substring."""
+
+ substrings: tuple[str, ...]
+
+
+class StorageHidesWhen(
+ msgspec.Struct,
+ frozen=True,
+ tag="storage",
+ tag_field="kind",
+ rename="camel",
+):
+ """Hide this suggestion when a live storage namespace matches."""
+
+ protocols: tuple[str, ...]
+ backend_types: tuple[str, ...]
+
+
+DetectedDataSourceHidesWhen = DialectHidesWhen | StorageHidesWhen
+
+
class DetectedDataSource(msgspec.Struct, frozen=True, rename="camel"):
"""A secret-free datasource suggestion produced by the kernel."""
@@ -62,3 +90,4 @@ class DetectedDataSource(msgspec.Struct, frozen=True, rename="camel"):
origins: tuple[DetectedDataSourceOrigin, ...]
configuration: tuple[DetectedDataSourceConfiguration, ...]
code: str
+ hides_when: DetectedDataSourceHidesWhen
diff --git a/marimo/_data/data_source_discovery/plugins/aws.py b/marimo/_data/data_source_discovery/plugins/aws.py
index e5ab7867f5..0c103ec914 100644
--- a/marimo/_data/data_source_discovery/plugins/aws.py
+++ b/marimo/_data/data_source_discovery/plugins/aws.py
@@ -6,6 +6,7 @@ from marimo._data.data_source_discovery.helpers import (
environment_variable,
has_all,
has_value,
+ hides_when_storage,
)
from marimo._data.data_source_discovery.models import (
DetectedDataSource,
@@ -62,6 +63,9 @@ def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
confidence="high",
origins=(ENVIRONMENT_ORIGIN,),
configuration=tuple(configuration),
+ hides_when=hides_when_storage(
+ protocols=("s3", "cloudflare", "coreweave"),
+ ),
code="""\
import s3fs
diff --git a/marimo/_data/data_source_discovery/plugins/databricks.py b/marimo/_data/data_source_discovery/plugins/databricks.py
index f33182c0a0..6593624eeb 100644
--- a/marimo/_data/data_source_discovery/plugins/databricks.py
+++ b/marimo/_data/data_source_discovery/plugins/databricks.py
@@ -5,6 +5,7 @@ from marimo._data.data_source_discovery.helpers import (
ENVIRONMENT_ORIGIN,
environment_variable,
has_value,
+ hides_when_dialect,
)
from marimo._data.data_source_discovery.models import DetectedDataSource
from marimo._data.data_source_discovery.types import (
@@ -63,6 +64,7 @@ connection = sql.connect(
origins=(ENVIRONMENT_ORIGIN,),
configuration=tuple(configuration),
code=code,
+ hides_when=hides_when_dialect("databricks"),
)
]
diff --git a/marimo/_data/data_source_discovery/plugins/huggingface.py b/marimo/_data/data_source_discovery/plugins/huggingface.py
index d7e1fe6315..4d6d459df8 100644
--- a/marimo/_data/data_source_discovery/plugins/huggingface.py
+++ b/marimo/_data/data_source_discovery/plugins/huggingface.py
@@ -5,6 +5,7 @@ from marimo._data.data_source_discovery.helpers import (
ENVIRONMENT_ORIGIN,
environment_variable,
has_value,
+ hides_when_storage,
)
from marimo._data.data_source_discovery.models import DetectedDataSource
from marimo._data.data_source_discovery.types import (
@@ -37,6 +38,10 @@ def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
confidence="high",
origins=(ENVIRONMENT_ORIGIN,),
configuration=(environment_variable("Token", token_name),),
+ hides_when=hides_when_storage(
+ protocols=("hf",),
+ backend_types=("huggingface",),
+ ),
code="""\
from huggingface_hub import HfApi
diff --git a/marimo/_data/data_source_discovery/plugins/mysql.py b/marimo/_data/data_source_discovery/plugins/mysql.py
index 7d4cca6f2f..614249bb49 100644
--- a/marimo/_data/data_source_discovery/plugins/mysql.py
+++ b/marimo/_data/data_source_discovery/plugins/mysql.py
@@ -6,6 +6,7 @@ from marimo._data.data_source_discovery.helpers import (
environment_variable,
has_all,
has_value,
+ hides_when_dialect,
is_valid_port,
)
from marimo._data.data_source_discovery.models import DetectedDataSource
@@ -66,6 +67,7 @@ engine = sqlalchemy.create_engine(DATABASE_URL)"""
origins=(ENVIRONMENT_ORIGIN,),
configuration=tuple(configuration),
code=code,
+ hides_when=hides_when_dialect("mysql"),
)
]
diff --git a/marimo/_data/data_source_discovery/plugins/postgres.py b/marimo/_data/data_source_discovery/plugins/postgres.py
index 44305bbf21..12e36cf528 100644
--- a/marimo/_data/data_source_discovery/plugins/postgres.py
+++ b/marimo/_data/data_source_discovery/plugins/postgres.py
@@ -6,6 +6,7 @@ from marimo._data.data_source_discovery.helpers import (
environment_variable,
has_all,
has_value,
+ hides_when_dialect,
is_valid_port,
)
from marimo._data.data_source_discovery.models import DetectedDataSource
@@ -70,6 +71,7 @@ def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
origins=(ENVIRONMENT_ORIGIN,),
configuration=tuple(configuration),
code=code,
+ hides_when=hides_when_dialect("postgres"),
)
]
diff --git a/marimo/_data/data_source_discovery/plugins/pyiceberg.py b/marimo/_data/data_source_discovery/plugins/pyiceberg.py
index 4f8191094d..71e678659a 100644
--- a/marimo/_data/data_source_discovery/plugins/pyiceberg.py
+++ b/marimo/_data/data_source_discovery/plugins/pyiceberg.py
@@ -14,6 +14,7 @@ from marimo._data.data_source_discovery.helpers import (
ENVIRONMENT_ORIGIN,
RESOLVED_CONFIGURATION_ORIGIN,
environment_variable,
+ hides_when_dialect,
safe_literal,
)
from marimo._data.data_source_discovery.models import (
@@ -281,6 +282,7 @@ def _iceberg_source(
confidence="high",
origins=origins,
configuration=tuple(configuration),
+ hides_when=hides_when_dialect("iceberg"),
code="\n".join(
[
"from pyiceberg.catalog import load_catalog",
@@ -313,6 +315,7 @@ def _merge_catalogs(
dict.fromkeys((*existing.configuration, *source.configuration))
),
code=existing.code,
+ hides_when=existing.hides_when,
)
return list(by_id.values())
diff --git a/marimo/_data/data_source_discovery/plugins/pyspark.py b/marimo/_data/data_source_discovery/plugins/pyspark.py
index 22a7016104..8fac0142bb 100644
--- a/marimo/_data/data_source_discovery/plugins/pyspark.py
+++ b/marimo/_data/data_source_discovery/plugins/pyspark.py
@@ -5,6 +5,7 @@ from marimo._data.data_source_discovery.helpers import (
ENVIRONMENT_ORIGIN,
environment_variable,
has_value,
+ hides_when_dialect,
)
from marimo._data.data_source_discovery.models import DetectedDataSource
from marimo._data.data_source_discovery.types import (
@@ -26,6 +27,7 @@ def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
confidence="high",
origins=(ENVIRONMENT_ORIGIN,),
configuration=(environment_variable("Remote", "SPARK_REMOTE"),),
+ hides_when=hides_when_dialect("pyspark", "spark"),
code="""\
import os
import ibis
diff --git a/marimo/_data/data_source_discovery/plugins/trino.py b/marimo/_data/data_source_discovery/plugins/trino.py
index b1fc360d4e..4bb523dec0 100644
--- a/marimo/_data/data_source_discovery/plugins/trino.py
+++ b/marimo/_data/data_source_discovery/plugins/trino.py
@@ -6,6 +6,7 @@ from marimo._data.data_source_discovery.helpers import (
environment_variable,
has_all,
has_value,
+ hides_when_dialect,
is_valid_port,
)
from marimo._data.data_source_discovery.models import DetectedDataSource
@@ -97,6 +98,7 @@ def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
origins=(ENVIRONMENT_ORIGIN,),
configuration=tuple(configuration),
code=code,
+ hides_when=hides_when_dialect("trino"),
)
]
diff --git a/marimo/_schemas/generated/notifications.yaml b/marimo/_schemas/generated/notifications.yaml
index 3e1b56c2fd..c8f13707e3 100644
--- a/marimo/_schemas/generated/notifications.yaml
+++ b/marimo/_schemas/generated/notifications.yaml
@@ -717,6 +717,15 @@ components:
type: array
displayName:
type: string
+ hidesWhen:
+ anyOf:
+ - $ref: '#/components/schemas/DialectHidesWhen'
+ - $ref: '#/components/schemas/StorageHidesWhen'
+ discriminator:
+ mapping:
+ dialect: '#/components/schemas/DialectHidesWhen'
+ storage: '#/components/schemas/StorageHidesWhen'
+ propertyName: kind
id:
type: string
integration:
@@ -734,6 +743,7 @@ components:
- origins
- configuration
- code
+ - hidesWhen
title: DetectedDataSource
type: object
DetectedDataSourceConfiguration:
@@ -767,6 +777,22 @@ components:
- label
title: DetectedDataSourceOrigin
type: object
+ DialectHidesWhen:
+ description: Hide this suggestion when a live SQL engine dialect contains a
+ substring.
+ properties:
+ kind:
+ enum:
+ - dialect
+ substrings:
+ items:
+ type: string
+ type: array
+ required:
+ - kind
+ - substrings
+ title: DialectHidesWhen
+ type: object
EnvironmentVariableDiscoveryValue:
description: A reference to an environment variable, never its value.
properties:
@@ -1628,6 +1654,26 @@ components:
- status
title: StartupLogsNotification
type: object
+ StorageHidesWhen:
+ description: Hide this suggestion when a live storage namespace matches.
+ properties:
+ backendTypes:
+ items:
+ type: string
+ type: array
+ kind:
+ enum:
+ - storage
+ protocols:
+ items:
+ type: string
+ type: array
+ required:
+ - kind
+ - protocols
+ - backendTypes
+ title: StorageHidesWhen
+ type: object
UIElementMessageNotification:
description: "Sends a message to a UI element/widget.\n\n Attributes:\n \
\ ui_element: UI element identifier.\n message: Message payload\
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 8c956d4313..93149a18a4 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -1530,6 +1530,15 @@ components:
type: array
displayName:
type: string
+ hidesWhen:
+ anyOf:
+ - $ref: '#/components/schemas/DialectHidesWhen'
+ - $ref: '#/components/schemas/StorageHidesWhen'
+ discriminator:
+ mapping:
+ dialect: '#/components/schemas/DialectHidesWhen'
+ storage: '#/components/schemas/StorageHidesWhen'
+ propertyName: kind
id:
type: string
integration:
@@ -1547,6 +1556,7 @@ components:
- origins
- configuration
- code
+ - hidesWhen
title: DetectedDataSource
type: object
DetectedDataSourceConfiguration:
@@ -1580,6 +1590,22 @@ components:
- label
title: DetectedDataSourceOrigin
type: object
+ DialectHidesWhen:
+ description: Hide this suggestion when a live SQL engine dialect contains a
+ substring.
+ properties:
+ kind:
+ enum:
+ - dialect
+ substrings:
+ items:
+ type: string
+ type: array
+ required:
+ - kind
+ - substrings
+ title: DialectHidesWhen
+ type: object
DiagnosticsConfig:
description: "Configuration options for diagnostics.\n\n **Keys.**\n\n \
\ - `enabled`: if `True`, diagnostics will be shown in the editor\n -\
@@ -5510,6 +5536,26 @@ components:
- lastModified
title: StorageEntry
type: object
+ StorageHidesWhen:
+ description: Hide this suggestion when a live storage namespace matches.
+ properties:
+ backendTypes:
+ items:
+ type: string
+ type: array
+ kind:
+ enum:
+ - storage
+ protocols:
+ items:
+ type: string
+ type: array
+ required:
+ - kind
+ - protocols
+ - backendTypes
+ title: StorageHidesWhen
+ type: object
StorageListEntriesCommand:
description: "List storage entries at a prefix.\n\n Navigates storage like\
\ a folder tree using delimiter-based listing.\n Returns entries (files/objects)\
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index 5b350d23f7..69006608a6 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -4611,6 +4611,9 @@ export interface components {
confidence: "high" | "medium";
configuration: components["schemas"]["DetectedDataSourceConfiguration"][];
displayName: string;
+ hidesWhen:
+ | components["schemas"]["DialectHidesWhen"]
+ | components["schemas"]["StorageHidesWhen"];
id: string;
integration: string;
origins: components["schemas"]["DetectedDataSourceOrigin"][];
@@ -4628,6 +4631,15 @@ export interface components {
/** @enum {unknown} */
type: "configuration" | "environment";
};
+ /**
+ * DialectHidesWhen
+ * @description Hide this suggestion when a live SQL engine dialect contains a substring.
+ */
+ DialectHidesWhen: {
+ /** @enum {unknown} */
+ kind: "dialect";
+ substrings: string[];
+ };
/**
* DiagnosticsConfig
* @description Configuration options for diagnostics.
@@ -7054,6 +7066,16 @@ export interface components {
path: string;
size: number;
};
+ /**
+ * StorageHidesWhen
+ * @description Hide this suggestion when a live storage namespace matches.
+ */
+ StorageHidesWhen: {
+ backendTypes: string[];
+ /** @enum {unknown} */
+ kind: "storage";
+ protocols: string[];
+ };
/**
* StorageListEntriesCommand
* @description List storage entries at a prefix.
diff --git a/tests/_data/data_source_discovery/test_aws.py b/tests/_data/data_source_discovery/test_aws.py
index 57b16beef0..7980eba757 100644
--- a/tests/_data/data_source_discovery/test_aws.py
+++ b/tests/_data/data_source_discovery/test_aws.py
@@ -73,6 +73,11 @@ import s3fs
# Uses the standard AWS credential and endpoint environment variables.
fs = s3fs.S3FileSystem()""",
+ "hidesWhen": {
+ "kind": "storage",
+ "protocols": ["s3", "cloudflare", "coreweave"],
+ "backendTypes": [],
+ },
}
]
)
diff --git a/tests/_data/data_source_discovery/test_databricks.py b/tests/_data/data_source_discovery/test_databricks.py
index 754da0da3d..2313d7c88b 100644
--- a/tests/_data/data_source_discovery/test_databricks.py
+++ b/tests/_data/data_source_discovery/test_databricks.py
@@ -67,6 +67,10 @@ connection = sql.connect(
"Authorization"
].removeprefix("Bearer "),
)""",
+ "hidesWhen": {
+ "kind": "dialect",
+ "substrings": ["databricks"],
+ },
}
]
)
diff --git a/tests/_data/data_source_discovery/test_discover.py b/tests/_data/data_source_discovery/test_discover.py
index 7b8b46dc07..8bf33703b5 100644
--- a/tests/_data/data_source_discovery/test_discover.py
+++ b/tests/_data/data_source_discovery/test_discover.py
@@ -13,6 +13,7 @@ from marimo._data.data_source_discovery import discover_data_sources
from marimo._data.data_source_discovery.discover import LOGGER
from marimo._data.data_source_discovery.helpers import (
ENVIRONMENT_ORIGIN,
+ hides_when_dialect,
)
from marimo._data.data_source_discovery.models import DetectedDataSource
from marimo._data.data_source_discovery.types import (
@@ -31,6 +32,7 @@ def test_isolates_failures_and_deduplicates() -> None:
origins=(ENVIRONMENT_ORIGIN,),
configuration=(),
code="connection = shared()",
+ hides_when=hides_when_dialect("example"),
)
duplicate = DetectedDataSource(
id="shared",
@@ -41,6 +43,7 @@ def test_isolates_failures_and_deduplicates() -> None:
origins=(ENVIRONMENT_ORIGIN,),
configuration=(),
code="connection = replacement()",
+ hides_when=hides_when_dialect("example"),
)
def fail(_context: DiscoveryContext) -> list[DetectedDataSource]:
diff --git a/tests/_data/data_source_discovery/test_discovery_huggingface.py b/tests/_data/data_source_discovery/test_discovery_huggingface.py
index 292966be79..c5a753c4c2 100644
--- a/tests/_data/data_source_discovery/test_discovery_huggingface.py
+++ b/tests/_data/data_source_discovery/test_discovery_huggingface.py
@@ -46,6 +46,11 @@ def test_discovers_hf_token() -> None:
from huggingface_hub import HfApi
hf = HfApi()""",
+ "hidesWhen": {
+ "kind": "storage",
+ "protocols": ["hf"],
+ "backendTypes": ["huggingface"],
+ },
}
]
)
diff --git a/tests/_data/data_source_discovery/test_discovery_pyiceberg.py b/tests/_data/data_source_discovery/test_discovery_pyiceberg.py
index 49523351d3..6f1d32b6b6 100644
--- a/tests/_data/data_source_discovery/test_discovery_pyiceberg.py
+++ b/tests/_data/data_source_discovery/test_discovery_pyiceberg.py
@@ -95,6 +95,7 @@ def test_merges_environment_with_resolved_configuration() -> None:
from pyiceberg.catalog import load_catalog
catalog = load_catalog("prod")""",
+ "hidesWhen": {"kind": "dialect", "substrings": ["iceberg"]},
},
{
"id": "pyiceberg-hive",
@@ -128,6 +129,7 @@ catalog = load_catalog("prod")""",
from pyiceberg.catalog import load_catalog
catalog = load_catalog("hive")""",
+ "hidesWhen": {"kind": "dialect", "substrings": ["iceberg"]},
},
]
)
diff --git a/tests/_data/data_source_discovery/test_mysql.py b/tests/_data/data_source_discovery/test_mysql.py
index 73c4fa162a..b76e9a08ec 100644
--- a/tests/_data/data_source_discovery/test_mysql.py
+++ b/tests/_data/data_source_discovery/test_mysql.py
@@ -86,6 +86,7 @@ DATABASE_URL = sqlalchemy.URL.create(
database=os.environ["MYSQL_DATABASE"],
)
engine = sqlalchemy.create_engine(DATABASE_URL)""",
+ "hidesWhen": {"kind": "dialect", "substrings": ["mysql"]},
}
]
)
diff --git a/tests/_data/data_source_discovery/test_postgres.py b/tests/_data/data_source_discovery/test_postgres.py
index 6c917061a2..2657d37c52 100644
--- a/tests/_data/data_source_discovery/test_postgres.py
+++ b/tests/_data/data_source_discovery/test_postgres.py
@@ -85,6 +85,7 @@ DATABASE_URL = sqlalchemy.URL.create(
database=os.environ["PGDATABASE"],
)
engine = sqlalchemy.create_engine(DATABASE_URL)""",
+ "hidesWhen": {"kind": "dialect", "substrings": ["postgres"]},
}
]
)
diff --git a/tests/_data/data_source_discovery/test_pyspark.py b/tests/_data/data_source_discovery/test_pyspark.py
index 3d9154bd0e..369bbf6534 100644
--- a/tests/_data/data_source_discovery/test_pyspark.py
+++ b/tests/_data/data_source_discovery/test_pyspark.py
@@ -56,6 +56,10 @@ session = SparkSession.builder.remote(
os.environ["SPARK_REMOTE"]
).getOrCreate()
con = ibis.pyspark.connect(session)""",
+ "hidesWhen": {
+ "kind": "dialect",
+ "substrings": ["pyspark", "spark"],
+ },
}
]
)
diff --git a/tests/_data/data_source_discovery/test_trino.py b/tests/_data/data_source_discovery/test_trino.py
index ebe65f2351..76eefdbce4 100644
--- a/tests/_data/data_source_discovery/test_trino.py
+++ b/tests/_data/data_source_discovery/test_trino.py
@@ -108,6 +108,7 @@ TRINO_URL = sqlalchemy.URL.create(
database=database,
)
engine = sqlalchemy.create_engine(TRINO_URL, connect_args=connect_args)""",
+ "hidesWhen": {"kind": "dialect", "substrings": ["trino"]},
}
]
)
@@ -180,6 +181,7 @@ TRINO_URL = sqlalchemy.URL.create(
database=database,
)
engine = sqlalchemy.create_engine(TRINO_URL)""",
+ "hidesWhen": {"kind": "dialect", "substrings": ["trino"]},
}
]
)