A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
add data config in the UI (#6878)
Shahmir Varqha12b174df72d8
diff --git a/frontend/src/components/app-config/ai-config.tsx b/frontend/src/components/app-config/ai-config.tsxindex 265a694a2b..7f28b9aef7 100644--- a/frontend/src/components/app-config/ai-config.tsx+++ b/frontend/src/components/app-config/ai-config.tsx@@ -73,14 +73,12 @@ import {import { Switch } from "../ui/switch";import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs";import { Tooltip } from "../ui/tooltip";-import { SettingSubtitle } from "./common";+import { formItemClasses, SettingSubtitle } from "./common";import { AWS_REGIONS } from "./constants";import { IncorrectModelId } from "./incorrect-model-id";import { IsOverridden } from "./is-overridden";import { MCPConfig } from "./mcp-config";-const formItemClasses = "flex flex-row items-center space-x-1 space-y-0";-interface AiConfigProps {form: UseFormReturn<UserConfig>;config: UserConfig;diff --git a/frontend/src/components/app-config/common.tsx b/frontend/src/components/app-config/common.tsxindex fd3bb5d94b..9033f51eeb 100644--- a/frontend/src/components/app-config/common.tsx+++ b/frontend/src/components/app-config/common.tsx@@ -4,6 +4,8 @@ import type { HTMLProps, PropsWithChildren } from "react";import type { SqlOutputType } from "@/core/config/config-schema";import { cn } from "@/utils/cn";+export const formItemClasses = "flex flex-row items-center space-x-1 space-y-0";+export const SettingTitle: React.FC<PropsWithChildren> = ({ children }) => {return (<div className="text-md font-semibold text-muted-foreground uppercase tracking-wide mb-1">@@ -36,6 +38,18 @@ export const SettingDescription: React.FC<PropsWithChildren> = ({return <p className="text-sm text-muted-foreground">{children}</p>;};+export const SettingGroup: React.FC<{+ title: string;+ children: React.ReactNode;+}> = ({ title, children }) => {+ return (+ <div className="flex flex-col gap-4 pb-4">+ <SettingSubtitle>{title}</SettingSubtitle>+ {children}+ </div>+ );+};+export const SQL_OUTPUT_SELECT_OPTIONS: {label: string;value: SqlOutputType;diff --git a/frontend/src/components/app-config/data-form.tsx b/frontend/src/components/app-config/data-form.tsxnew file mode 100644index 0000000000..ba47055b2e--- /dev/null+++ b/frontend/src/components/app-config/data-form.tsx@@ -0,0 +1,282 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { ChangeEvent } from "react";+import type { FieldPath, UseFormReturn } from "react-hook-form";+import { Checkbox } from "@/components/ui/checkbox";+import {+ FormControl,+ FormDescription,+ FormField,+ FormItem,+ FormLabel,+ FormMessage,+} from "@/components/ui/form";+import { NativeSelect } from "@/components/ui/native-select";+import { NumberField } from "@/components/ui/number-field";+import type { UserConfig } from "@/core/config/config-schema";+import {+ formItemClasses,+ SettingGroup,+ SQL_OUTPUT_SELECT_OPTIONS,+} from "./common";+import { IsOverridden } from "./is-overridden";++const DISCOVERY_OPTIONS = ["auto", "true", "false"];++export const DataForm = ({+ form,+ config,+ onSubmit,+}: {+ form: UseFormReturn<UserConfig>;+ config: UserConfig;+ onSubmit: (values: UserConfig) => void;+}) => {+ const renderDiscoveryForm = (name: FieldPath<UserConfig>, label: string) => {+ return (+ <FormField+ control={form.control}+ name={name}+ render={({ field }) => {+ const onChange = (e: ChangeEvent<HTMLSelectElement>) => {+ const value = e.target.value;+ field.onChange(+ value === "true" ? true : value === "false" ? false : value,+ );+ };+ return (+ <FormItem className={formItemClasses}>+ <FormLabel className="text-sm font-normal w-16">+ {label}+ </FormLabel>+ <FormControl>+ <NativeSelect+ data-testid="auto-discover-schemas-select"+ onChange={onChange}+ value={+ field.value === undefined ? "auto" : field.value.toString()+ }+ disabled={field.disabled}+ className="w-[100px]"+ >+ {DISCOVERY_OPTIONS.map((option) => (+ <option value={option} key={option}>+ {option}+ </option>+ ))}+ </NativeSelect>+ </FormControl>+ <IsOverridden userConfig={config} name={name} />+ </FormItem>+ );+ }}+ />+ );+ };++ return (+ <>+ <FormField+ control={form.control}+ name="display.dataframes"+ render={({ field }) => (+ <div className="flex flex-col space-y-1">+ <FormItem className={formItemClasses}>+ <FormLabel>Dataframe viewer</FormLabel>+ <FormControl>+ <NativeSelect+ data-testid="display-dataframes-select"+ onChange={(e) => field.onChange(e.target.value)}+ value={field.value}+ disabled={field.disabled}+ className="inline-flex mr-2"+ >+ {["rich", "plain"].map((option) => (+ <option value={option} key={option}>+ {option}+ </option>+ ))}+ </NativeSelect>+ </FormControl>+ <FormMessage />+ <IsOverridden userConfig={config} name="display.dataframes" />+ </FormItem>++ <FormDescription>+ Whether to use marimo's rich dataframe viewer or a plain HTML+ table. This requires restarting your notebook to take effect.+ </FormDescription>+ </div>+ )}+ />+ <FormField+ control={form.control}+ name="display.default_table_page_size"+ render={({ field }) => (+ <div className="flex flex-col space-y-1">+ <FormItem className={formItemClasses}>+ <FormLabel>Default table page size</FormLabel>+ <FormControl>+ <NumberField+ aria-label="Default table page size"+ data-testid="default-table-page-size-input"+ className="m-0 w-24"+ {...field}+ value={field.value}+ minValue={1}+ step={1}+ onChange={(value) => {+ field.onChange(value);+ if (!Number.isNaN(value)) {+ onSubmit(form.getValues());+ }+ }}+ />+ </FormControl>+ <FormMessage />+ <IsOverridden+ userConfig={config}+ name="display.default_table_page_size"+ />+ </FormItem>+ <FormDescription>+ The default number of rows displayed in dataframes and SQL+ results.+ </FormDescription>+ </div>+ )}+ />+ <FormField+ control={form.control}+ name="display.default_table_max_columns"+ render={({ field }) => (+ <div className="flex flex-col space-y-1">+ <FormItem className={formItemClasses}>+ <FormLabel>Default table max columns</FormLabel>+ <FormControl>+ <NumberField+ aria-label="Default table max columns"+ data-testid="default-table-max-columns-input"+ className="m-0 w-24"+ {...field}+ value={field.value}+ minValue={1}+ step={1}+ onChange={(value) => {+ field.onChange(value);+ if (!Number.isNaN(value)) {+ onSubmit(form.getValues());+ }+ }}+ />+ </FormControl>+ <FormMessage />+ <IsOverridden+ userConfig={config}+ name="display.default_table_max_columns"+ />+ </FormItem>+ <FormDescription>+ The default maximum number of columns displayed in dataframes and+ SQL results.+ </FormDescription>+ </div>+ )}+ />++ <SettingGroup title="SQL">+ <div className="flex flex-col gap-1">+ <div className="text-sm text-foreground">+ Database Schema Discovery+ </div>+ <div className="text-sm text-muted-foreground mb-2">+ Whether database schemas, tables, and columns are automatically+ discovered.+ <br />+ <span className="font-semibold">+ Can be expensive for large databases.+ </span>{" "}+ Use 'auto' to determine introspection based on the{" "}+ <a+ className="text-link hover:underline"+ rel="noopener noreferrer"+ target="_blank"+ href="https://docs.marimo.io/guides/working_with_data/sql/?h=database#database-schema-and-table-auto-discovery"+ >+ database+ </a>+ .+ </div>++ {renderDiscoveryForm("datasources.auto_discover_schemas", "Schemas")}+ {renderDiscoveryForm("datasources.auto_discover_tables", "Tables")}+ {renderDiscoveryForm("datasources.auto_discover_columns", "Columns")}+ </div>++ <FormField+ control={form.control}+ name="diagnostics.sql_linter"+ render={({ field }) => (+ <div className="flex flex-col space-y-1">+ <FormItem className={formItemClasses}>+ <FormLabel>SQL Linter</FormLabel>+ <FormControl>+ <Checkbox+ data-testid="sql-linter-checkbox"+ checked={field.value}+ onCheckedChange={field.onChange}+ />+ </FormControl>+ <FormMessage />+ <IsOverridden+ userConfig={config}+ name="diagnostics.sql_linter"+ />+ </FormItem>+ <FormDescription>+ Better linting and autocompletions for SQL cells.+ </FormDescription>+ </div>+ )}+ />++ <FormField+ control={form.control}+ name="runtime.default_sql_output"+ render={({ field }) => (+ <div className="flex flex-col space-y-1">+ <FormItem className={formItemClasses}>+ <FormLabel>Default SQL output</FormLabel>+ <FormControl>+ <NativeSelect+ data-testid="user-config-sql-output-select"+ onChange={(e) => field.onChange(e.target.value)}+ value={field.value}+ disabled={field.disabled}+ className="inline-flex mr-2"+ >+ {SQL_OUTPUT_SELECT_OPTIONS.map((option) => (+ <option value={option.value} key={option.value}>+ {option.label}+ </option>+ ))}+ </NativeSelect>+ </FormControl>+ <FormMessage />+ <IsOverridden+ userConfig={config}+ name="runtime.default_sql_output"+ />+ </FormItem>++ <FormDescription>+ The default SQL output type for new notebooks; overridden by+ "sql_output" in the application config.+ </FormDescription>+ </div>+ )}+ />+ </SettingGroup>+ </>+ );+};diff --git a/frontend/src/components/app-config/user-config-form.tsx b/frontend/src/components/app-config/user-config-form.tsxindex 906521d1ac..657db33d13 100644--- a/frontend/src/components/app-config/user-config-form.tsx+++ b/frontend/src/components/app-config/user-config-form.tsx@@ -9,8 +9,8 @@ import {EditIcon,FlaskConicalIcon,FolderCog2,+ LayersIcon,MonitorIcon,- PackageIcon,} from "lucide-react";import React, { useId, useRef } from "react";import { useLocale } from "react-aria";@@ -53,11 +53,11 @@ import { Badge } from "../ui/badge";import { ExternalLink } from "../ui/links";import { Tooltip } from "../ui/tooltip";import { AiConfig } from "./ai-config";-import { SettingSubtitle, SQL_OUTPUT_SELECT_OPTIONS } from "./common";+import { formItemClasses, SettingGroup } from "./common";+import { DataForm } from "./data-form";import { IsOverridden } from "./is-overridden";import { OptionalFeatures } from "./optional-features";-const formItemClasses = "flex flex-row items-center space-x-1 space-y-0";const categories = [{id: "editor",@@ -72,9 +72,9 @@ const categories = [className: "bg-(--grass-4)",},{- id: "packageManagement",- label: "Package Management",- Icon: PackageIcon,+ id: "packageManagementAndData",+ label: "Packages & Data",+ Icon: LayersIcon,className: "bg-(--red-4)",},{@@ -559,35 +559,6 @@ export const UserConfigForm: React.FC = () => {/></SettingGroup>- <SettingGroup title="Diagnostics">- <FormField- control={form.control}- name="diagnostics.sql_linter"- render={({ field }) => (- <div className="flex flex-col space-y-1">- <FormItem className={formItemClasses}>- <FormLabel>SQL Linter</FormLabel>- <FormControl>- <Checkbox- data-testid="sql-linter-checkbox"- checked={field.value}- onCheckedChange={field.onChange}- />- </FormControl>- <FormMessage />- <IsOverridden- userConfig={config}- name="diagnostics.sql_linter"- />- </FormItem>- <FormDescription>- Better linting and autocompletions for SQL cells.- </FormDescription>- </div>- )}- />- </SettingGroup>-<SettingGroup title="Keymap"><FormFieldcontrol={form.control}@@ -887,22 +858,30 @@ export const UserConfigForm: React.FC = () => {</div>)}/>+ </SettingGroup>+ </>+ );+ case "packageManagementAndData":+ return (+ <>+ <SettingGroup title="Package Management"><FormFieldcontrol={form.control}- name="display.dataframes"+ disabled={isWasmRuntime}+ name="package_management.manager"render={({ field }) => (<div className="flex flex-col space-y-1"><FormItem className={formItemClasses}>- <FormLabel>Dataframe viewer</FormLabel>+ <FormLabel>Manager</FormLabel><FormControl><NativeSelect- data-testid="display-dataframes-select"+ data-testid="package-manager-select"onChange={(e) => field.onChange(e.target.value)}value={field.value}disabled={field.disabled}className="inline-flex mr-2">- {["rich", "plain"].map((option) => (+ {PackageManagerNames.map((option) => (<option value={option} key={option}>{option}</option>@@ -912,149 +891,35 @@ export const UserConfigForm: React.FC = () => {<FormMessage /><IsOverriddenuserConfig={config}- name="display.dataframes"+ name="package_management.manager"/></FormItem><FormDescription>- Whether to use marimo's rich dataframe viewer or a plain- HTML table. This requires restarting your notebook to take- effect.- </FormDescription>- </div>- )}- />- <FormField- control={form.control}- name="display.default_table_page_size"- render={({ field }) => (- <div className="flex flex-col space-y-1">- <FormItem className={formItemClasses}>- <FormLabel>Default table page size</FormLabel>- <FormControl>- <NumberField- aria-label="Default table page size"- data-testid="default-table-page-size-input"- className="m-0 w-24"- {...field}- value={field.value}- minValue={1}- step={1}- onChange={(value) => {- field.onChange(value);- if (!Number.isNaN(value)) {- onSubmit(form.getValues());- }- }}- />- </FormControl>- <FormMessage />- <IsOverridden- userConfig={config}- name="display.default_table_page_size"- />- </FormItem>- <FormDescription>- The default number of rows displayed in dataframes and SQL- results.- </FormDescription>- </div>- )}- />- <FormField- control={form.control}- name="display.default_table_max_columns"- render={({ field }) => (- <div className="flex flex-col space-y-1">- <FormItem className={formItemClasses}>- <FormLabel>Default table max columns</FormLabel>- <FormControl>- <NumberField- aria-label="Default table max columns"- data-testid="default-table-max-columns-input"- className="m-0 w-24"- {...field}- value={field.value}- minValue={1}- step={1}- onChange={(value) => {- field.onChange(value);- if (!Number.isNaN(value)) {- onSubmit(form.getValues());- }- }}- />- </FormControl>- <FormMessage />- <IsOverridden- userConfig={config}- name="display.default_table_max_columns"- />- </FormItem>- <FormDescription>- The default maximum number of columns displayed in- dataframes and SQL results.+ When marimo comes across a module that is not installed,+ you will be prompted to install it using your preferred+ package manager. Learn more in the{" "}+ <ExternalLink href="https://docs.marimo.io/guides/editor_features/package_management.html">+ docs+ </ExternalLink>+ .+ <br />+ <br />+ Running marimo in a{" "}+ <ExternalLink href="https://docs.marimo.io/guides/editor_features/package_management.html#running-marimo-in-a-sandbox-environment-uv-only">+ sandboxed environment+ </ExternalLink>{" "}+ is only supported by <Kbd className="inline">uv</Kbd></FormDescription></div>)}/></SettingGroup>+ <SettingGroup title="Data">+ <DataForm form={form} config={config} onSubmit={onSubmit} />+ </SettingGroup></>);- case "packageManagement":- return (- <SettingGroup title="Package Management">- <FormField- control={form.control}- disabled={isWasmRuntime}- name="package_management.manager"- render={({ field }) => (- <div className="flex flex-col space-y-1">- <FormItem className={formItemClasses}>- <FormLabel>Manager</FormLabel>- <FormControl>- <NativeSelect- data-testid="package-manager-select"- onChange={(e) => field.onChange(e.target.value)}- value={field.value}- disabled={field.disabled}- className="inline-flex mr-2"- >- {PackageManagerNames.map((option) => (- <option value={option} key={option}>- {option}- </option>- ))}- </NativeSelect>- </FormControl>- <FormMessage />- <IsOverridden- userConfig={config}- name="package_management.manager"- />- </FormItem>-- <FormDescription>- When marimo comes across a module that is not installed, you- will be prompted to install it using your preferred package- manager. Learn more in the{" "}- <ExternalLink href="https://docs.marimo.io/guides/editor_features/package_management.html">- docs- </ExternalLink>- .- <br />- <br />- Running marimo in a{" "}- <ExternalLink href="https://docs.marimo.io/guides/editor_features/package_management.html#running-marimo-in-a-sandbox-environment-uv-only">- sandboxed environment- </ExternalLink>{" "}- is only supported by <Kbd className="inline">uv</Kbd>- </FormDescription>- </div>- )}- />- </SettingGroup>- );case "runtime":return (<SettingGroup title="Runtime configuration">@@ -1167,43 +1032,6 @@ export const UserConfigForm: React.FC = () => {)}/>- <FormField- control={form.control}- name="runtime.default_sql_output"- render={({ field }) => (- <div className="flex flex-col space-y-1">- <FormItem className={formItemClasses}>- <FormLabel>Default SQL output</FormLabel>- <FormControl>- <NativeSelect- data-testid="user-config-sql-output-select"- onChange={(e) => field.onChange(e.target.value)}- value={field.value}- disabled={field.disabled}- className="inline-flex mr-2"- >- {SQL_OUTPUT_SELECT_OPTIONS.map((option) => (- <option value={option.value} key={option.value}>- {option.label}- </option>- ))}- </NativeSelect>- </FormControl>- <FormMessage />- <IsOverridden- userConfig={config}- name="runtime.default_sql_output"- />- </FormItem>-- <FormDescription>- The default SQL output type for new notebooks; overridden by- "sql_output" in the application config.- </FormDescription>- </div>- )}- />-<FormFieldcontrol={form.control}name="runtime.reactive_tests"@@ -1434,18 +1262,3 @@ export const UserConfigForm: React.FC = () => {</Form>);};--const SettingGroup = ({- title,- children,-}: {- title: string;- children: React.ReactNode;-}) => {- return (- <div className="flex flex-col gap-4 pb-4">- <SettingSubtitle>{title}</SettingSubtitle>- {children}- </div>- );-};diff --git a/frontend/src/components/editor/chrome/panels/packages-panel.tsx b/frontend/src/components/editor/chrome/panels/packages-panel.tsxindex 0e3a80b567..f8eacd55f4 100644--- a/frontend/src/components/editor/chrome/panels/packages-panel.tsx+++ b/frontend/src/components/editor/chrome/panels/packages-panel.tsx@@ -211,7 +211,7 @@ const InstallPackageForm: React.FC<{) : (<Tooltip content="Change package manager"><BoxIcon- onClick={() => openSettings("packageManagement")}+ onClick={() => openSettings("packageManagementAndData")}className="mr-2 h-4 w-4 shrink-0 opacity-50 hover:opacity-80 cursor-pointer"/></Tooltip>diff --git a/frontend/src/components/editor/package-alert.tsx b/frontend/src/components/editor/package-alert.tsxindex 1a904df1b9..4a5ce0186a 100644--- a/frontend/src/components/editor/package-alert.tsx+++ b/frontend/src/components/editor/package-alert.tsx@@ -118,44 +118,46 @@ export const PackageAlert: React.FC = () => {<div><p>The following packages were not found:</p><table className="list-disc ml-2 mt-1">- {packageAlert.packages.map((pkg, index) => {- const parsed = parsePackageSpecifier(pkg);- const currentExtras = selectedExtras[pkg] || parsed.extras;-- return (- <tr className="font-mono text-sm" key={index}>- <td className="pr-2 py-1 align-middle">- <BoxIcon size="1rem" />- </td>- <td className="pr-2 py-1 align-middle">- <ExtrasSelector- packageName={parsed.name}- selectedExtras={currentExtras}- onExtrasChange={(extras) =>- setSelectedExtras((prev) => ({- ...prev,- [pkg]: extras,- }))- }- />- </td>- {doesSupportVersioning && (+ <tbody>+ {packageAlert.packages.map((pkg, index) => {+ const parsed = parsePackageSpecifier(pkg);+ const currentExtras = selectedExtras[pkg] || parsed.extras;++ return (+ <tr className="font-mono text-sm" key={index}>+ <td className="pr-2 py-1 align-middle">+ <BoxIcon size="1rem" />+ </td><td className="pr-2 py-1 align-middle">- <PackageVersionSelect- value={desiredPackageVersions[pkg] ?? "latest"}- onChange={(value) =>- setDesiredPackageVersions((prev) => ({+ <ExtrasSelector+ packageName={parsed.name}+ selectedExtras={currentExtras}+ onExtrasChange={(extras) =>+ setSelectedExtras((prev) => ({...prev,- [pkg]: value,+ [pkg]: extras,}))}- packageName={parsed.name}/></td>- )}- </tr>- );- })}+ {doesSupportVersioning && (+ <td className="pr-2 py-1 align-middle">+ <PackageVersionSelect+ value={desiredPackageVersions[pkg] ?? "latest"}+ onChange={(value) =>+ setDesiredPackageVersions((prev) => ({+ ...prev,+ [pkg]: value,+ }))+ }+ packageName={parsed.name}+ />+ </td>+ )}+ </tr>+ );+ })}+ </tbody></table></div><div className="ml-auto flex flex-row items-baseline">diff --git a/marimo/_sql/engines/ibis.py b/marimo/_sql/engines/ibis.pyindex aeb53d89ef..78efa7d3c0 100644--- a/marimo/_sql/engines/ibis.py+++ b/marimo/_sql/engines/ibis.py@@ -13,7 +13,7 @@ from marimo._data.models import ()from marimo._dependencies.dependencies import DependencyManagerfrom marimo._sql.engines.types import InferenceConfig, SQLConnection-from marimo._sql.utils import convert_to_output+from marimo._sql.utils import CHEAP_DISCOVERY_DATABASES, convert_to_outputfrom marimo._types.ids import VariableNameif TYPE_CHECKING:@@ -474,9 +474,4 @@ class IbisEngine(SQLConnection["SQLBackend"]):return valuedef _is_cheap_discovery(self) -> bool:- return self.dialect.lower() in (- "duckdb",- "sqlite",- "mysql",- "postgresql",- )+ return self.dialect.lower() in CHEAP_DISCOVERY_DATABASESdiff --git a/marimo/_sql/engines/sqlalchemy.py b/marimo/_sql/engines/sqlalchemy.pyindex f083285934..eb72d9d94f 100644--- a/marimo/_sql/engines/sqlalchemy.py+++ b/marimo/_sql/engines/sqlalchemy.py@@ -14,7 +14,11 @@ from marimo._data.models import ()from marimo._dependencies.dependencies import DependencyManagerfrom marimo._sql.engines.types import InferenceConfig, SQLConnection-from marimo._sql.utils import convert_to_output, sql_type_to_data_type+from marimo._sql.utils import (+ CHEAP_DISCOVERY_DATABASES,+ convert_to_output,+ sql_type_to_data_type,+)from marimo._types.ids import VariableNameLOGGER = _loggers.marimo_logger()@@ -422,7 +426,7 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):return valuedef _is_cheap_discovery(self) -> bool:- return self.dialect.lower() in ("sqlite", "mysql", "postgresql")+ return self.dialect.lower() in CHEAP_DISCOVERY_DATABASES@staticmethoddef is_cursor_result(result: Any) -> bool:diff --git a/marimo/_sql/utils.py b/marimo/_sql/utils.pyindex 3671c16401..de4a347cae 100644--- a/marimo/_sql/utils.py+++ b/marimo/_sql/utils.py@@ -21,6 +21,8 @@ if TYPE_CHECKING:LOGGER = _loggers.marimo_logger()+CHEAP_DISCOVERY_DATABASES = ["duckdb", "sqlite", "mysql", "postgresql"]+def wrapped_sql(query: str,