Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: preserve Arrow geometry exports and guard stale retries (#11335)

Kiran Gadhaveaba36a1cb066
diff --git a/frontend/src/components/data-table/__tests__/export-actions.test.tsx b/frontend/src/components/data-table/__tests__/export-actions.test.tsx
index e009fd378a..4bb21035d4 100644
--- a/frontend/src/components/data-table/__tests__/export-actions.test.tsx
+++ b/frontend/src/components/data-table/__tests__/export-actions.test.tsx
@@ -54,6 +54,7 @@ function renderExportActions(
props: Partial<ExportActionProps> = {},
sizeLimit: { limitBytes: number; unavailableMessage: string } | null = null,
mode: AppMode = "edit",
+ strictMode = false,
) {
const store = createStore();
store.set(downloadSizeLimitAtom, sizeLimit);
@@ -66,7 +67,7 @@ function renderExportActions(
</TooltipProvider>
</Provider>
);
- const result = render(view(props));
+ const result = render(view(props), { reactStrictMode: strictMode });
return {
...result,
rerenderExportActions: (nextProps: Partial<ExportActionProps>) =>
@@ -284,7 +285,7 @@ describe("ExportActions dialog", () => {
await waitFor(() => {
expect(downloadByURL).toHaveBeenCalled();
});
- expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ expect(getExportMetadata).toHaveBeenCalledTimes(3);
expect(
screen.queryByText("GeoParquet export requires pyarrow."),
).not.toBeInTheDocument();
@@ -310,10 +311,10 @@ describe("ExportActions dialog", () => {
mocks.handleInstallPackages.mock.calls[0][1]();
});
await waitFor(() => {
- expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ expect(getExportMetadata).toHaveBeenCalledTimes(3);
});
expect(
- screen.getByRole("button", { name: "Install pyarrow" }),
+ await screen.findByRole("button", { name: "Install pyarrow" }),
).toBeInTheDocument();
expect(
screen.getByRole("button", { name: "Download GeoParquet" }),
@@ -530,6 +531,328 @@ describe("ExportActions dialog", () => {
expect(downloadByURL).not.toHaveBeenCalled();
});
+ async function startGeoParquetInstall(strictMode = false) {
+ const getExportMetadata = vi.fn().mockResolvedValue(geometryMetadata);
+ downloadAs.mockResolvedValueOnce({
+ url: "",
+ filename: "",
+ error: "GeoParquet export requires pyarrow.",
+ missing_packages: ["pyarrow"],
+ });
+ const { rerenderExportActions, unmount } = renderExportActions(
+ { getExportMetadata, metadataSource: "first" },
+ null,
+ "edit",
+ strictMode,
+ );
+ const dialog = await openDialog();
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Download GeoParquet" }),
+ );
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Install pyarrow" }),
+ );
+ return { dialog, getExportMetadata, rerenderExportActions, unmount };
+ }
+
+ it("does not retry with a geometry choice after the final geometry disappears", async () => {
+ const { getExportMetadata } = await startGeoParquetInstall();
+ getExportMetadata.mockResolvedValue({
+ geometry_columns: [],
+ primary_geometry_column: null,
+ default_geometry_column: null,
+ formats: {},
+ });
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+
+ expect(downloadAs).toHaveBeenCalledTimes(1);
+ expect(downloadByURL).not.toHaveBeenCalled();
+ expect(
+ await screen.findByText(
+ "The selected geometry is no longer available. Choose another geometry to export.",
+ ),
+ ).toBeInTheDocument();
+ fireEvent.click(screen.getByRole("button", { name: "Download Parquet" }));
+ await waitFor(() => expect(downloadByURL).toHaveBeenCalledTimes(1));
+ expect(downloadAs).toHaveBeenLastCalledWith({ format: "parquet" });
+ });
+
+ it("does not export when installation finishes after the widget unmounts", async () => {
+ const { getExportMetadata, unmount } = await startGeoParquetInstall();
+ unmount();
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+
+ expect(getExportMetadata).toHaveBeenCalledTimes(1);
+ expect(downloadAs).toHaveBeenCalledTimes(1);
+ expect(downloadByURL).not.toHaveBeenCalled();
+ });
+
+ it("does not export when post-install metadata resolves after unmount", async () => {
+ const { getExportMetadata, unmount } = await startGeoParquetInstall();
+ let resolveCheck: (value: ExportMetadata) => void = () => undefined;
+ getExportMetadata.mockImplementationOnce(
+ () =>
+ new Promise<ExportMetadata>((resolve) => {
+ resolveCheck = resolve;
+ }),
+ );
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+ expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ unmount();
+
+ await act(async () => resolveCheck(geometryMetadata));
+
+ expect(downloadAs).toHaveBeenCalledTimes(1);
+ expect(downloadByURL).not.toHaveBeenCalled();
+ });
+
+ it("does not deliver a retried export that resolves after unmount", async () => {
+ const { unmount } = await startGeoParquetInstall();
+ let resolveDownload: (value: {
+ url: string;
+ filename: string;
+ }) => void = () => undefined;
+ downloadAs.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ resolveDownload = resolve;
+ }),
+ );
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+ expect(downloadAs).toHaveBeenCalledTimes(2);
+ unmount();
+
+ await act(async () =>
+ resolveDownload({
+ url: "https://example.test/export",
+ filename: "table",
+ }),
+ );
+
+ expect(downloadByURL).not.toHaveBeenCalled();
+ });
+
+ it("retries after installation under StrictMode effect replay", async () => {
+ await startGeoParquetInstall(true);
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+
+ await waitFor(() => expect(downloadByURL).toHaveBeenCalledTimes(1));
+ expect(downloadAs).toHaveBeenLastCalledWith({
+ format: "parquet",
+ geometry_column: "location",
+ });
+ });
+
+ it("does not export when installation finishes after the dialog closes", async () => {
+ const { dialog, getExportMetadata } = await startGeoParquetInstall();
+ fireEvent.click(within(dialog).getByRole("button", { name: "Close" }));
+ await waitFor(() => {
+ expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
+ });
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+
+ expect(getExportMetadata).toHaveBeenCalledTimes(1);
+ expect(downloadAs).toHaveBeenCalledTimes(1);
+ expect(downloadByURL).not.toHaveBeenCalled();
+ });
+
+ it("does not export when installation finishes after the source changes", async () => {
+ const { getExportMetadata, rerenderExportActions } =
+ await startGeoParquetInstall();
+ rerenderExportActions({ getExportMetadata, metadataSource: "second" });
+ await waitFor(() => {
+ expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ });
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+
+ expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ expect(downloadAs).toHaveBeenCalledTimes(1);
+ expect(downloadByURL).not.toHaveBeenCalled();
+ });
+
+ it("does not export a non-geographic retry after the dialog closes", async () => {
+ downloadAs.mockResolvedValueOnce({
+ url: "",
+ filename: "",
+ error: "Parquet export requires pyarrow.",
+ missing_packages: ["pyarrow"],
+ });
+ renderExportActions();
+ const dialog = await openDialog();
+ fireEvent.click(screen.getByRole("button", { name: "Download Parquet" }));
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Install pyarrow" }),
+ );
+ fireEvent.click(within(dialog).getByRole("button", { name: "Close" }));
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+
+ expect(downloadAs).toHaveBeenCalledTimes(1);
+ expect(downloadByURL).not.toHaveBeenCalled();
+ });
+
+ it("retries after installation when an earlier export succeeded", async () => {
+ const getExportMetadata = vi.fn().mockResolvedValue({
+ ...geometryMetadata,
+ formats: {
+ parquet: {
+ available: false,
+ reason: "GeoParquet export requires pyarrow.",
+ missing_packages: ["pyarrow"],
+ },
+ },
+ });
+ renderExportActions({ getExportMetadata });
+ await openDialog();
+ fireEvent.click(screen.getByRole("button", { name: "Download CSV" }));
+ await waitFor(() => expect(downloadByURL).toHaveBeenCalledTimes(1));
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Install pyarrow" }),
+ );
+ getExportMetadata.mockResolvedValue(geometryMetadata);
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+
+ await waitFor(() => expect(downloadByURL).toHaveBeenCalledTimes(2));
+ expect(downloadAs).toHaveBeenLastCalledWith({
+ format: "parquet",
+ geometry_column: "location",
+ });
+ });
+
+ it("waits for the post-install metadata check before exporting", async () => {
+ let resolveCheck: (value: ExportMetadata) => void = () => undefined;
+ const getExportMetadata = vi.fn().mockResolvedValue({
+ ...geometryMetadata,
+ formats: {
+ parquet: {
+ available: false,
+ reason: "GeoParquet export requires pyarrow.",
+ missing_packages: ["pyarrow"],
+ },
+ },
+ });
+ renderExportActions({ getExportMetadata });
+ await openDialog();
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Install pyarrow" }),
+ );
+ getExportMetadata.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ resolveCheck = resolve;
+ }),
+ );
+
+ await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
+ expect(downloadAs).not.toHaveBeenCalled();
+
+ getExportMetadata.mockResolvedValue(geometryMetadata);
+ await act(async () => resolveCheck(geometryMetadata));
+ await waitFor(() => expect(downloadByURL).toHaveBeenCalled());
+ expect(downloadAs).toHaveBeenCalledTimes(1);
+ });
+
+ it("uses the new source eligibility after a settled package failure", async () => {
+ const getExportMetadata = vi.fn().mockResolvedValue(geometryMetadata);
+ downloadAs.mockResolvedValueOnce({
+ url: "",
+ filename: "",
+ error: "GeoParquet export requires pyarrow.",
+ missing_packages: ["pyarrow"],
+ });
+ const { rerenderExportActions } = renderExportActions({
+ getExportMetadata,
+ metadataSource: "first",
+ });
+ await openDialog();
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Download GeoParquet" }),
+ );
+ await screen.findByRole("button", { name: "Install pyarrow" });
+
+ rerenderExportActions({ getExportMetadata, metadataSource: "second" });
+
+ await waitFor(() =>
+ expect(
+ screen.getByRole("button", { name: "Download GeoParquet" }),
+ ).toBeEnabled(),
+ );
+ expect(
+ screen.queryByRole("button", { name: "Install pyarrow" }),
+ ).not.toBeInTheDocument();
+ });
+
+ it("ignores metadata for an earlier source that resolves late", async () => {
+ let resolveFirst: (value: ExportMetadata) => void = () => undefined;
+ const first = vi.fn(
+ () =>
+ new Promise<ExportMetadata>((resolve) => {
+ resolveFirst = resolve;
+ }),
+ );
+ const second = vi.fn().mockResolvedValue({
+ ...geometryMetadata,
+ geometry_columns: [
+ { name: "region", encoding: "objects", crs: "EPSG:3857" },
+ ],
+ primary_geometry_column: "region",
+ default_geometry_column: "region",
+ });
+ const { rerenderExportActions } = renderExportActions({
+ getExportMetadata: first,
+ metadataSource: "first",
+ });
+ await openDialog();
+ rerenderExportActions({
+ getExportMetadata: second,
+ metadataSource: "second",
+ });
+ await waitFor(() =>
+ expect(screen.getByTestId("export-summary-parquet")).toHaveTextContent(
+ "region",
+ ),
+ );
+
+ await act(async () => resolveFirst(geometryMetadata));
+
+ expect(screen.getByTestId("export-summary-parquet")).toHaveTextContent(
+ "region",
+ );
+ });
+
+ it("does not use earlier metadata when the new source request fails", async () => {
+ const getExportMetadata = vi.fn().mockResolvedValue(geometryMetadata);
+ const { rerenderExportActions } = renderExportActions({
+ getExportMetadata,
+ metadataSource: "first",
+ });
+ await openDialog();
+ await waitFor(() =>
+ expect(
+ screen.getByRole("button", { name: "Download GeoParquet" }),
+ ).toBeEnabled(),
+ );
+ getExportMetadata.mockRejectedValueOnce(new Error("Connection lost"));
+
+ rerenderExportActions({ getExportMetadata, metadataSource: "second" });
+
+ expect(
+ await screen.findByText(
+ "Could not load geometry options: Connection lost",
+ ),
+ ).toBeInTheDocument();
+ expect(
+ screen.getByRole("button", { name: "Download Parquet" }),
+ ).toBeDisabled();
+ expect(screen.getByRole("button", { name: "Download CSV" })).toBeEnabled();
+ });
+
it("shows formats in the approved order with an options toggle where supported", async () => {
renderExportActions();
const dialog = await openDialog();
@@ -1412,7 +1735,7 @@ describe("ExportActions GeoJSON", () => {
);
await act(async () => mocks.handleInstallPackages.mock.calls[0][1]());
await waitFor(() => expect(downloadByURL).toHaveBeenCalled());
- expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ expect(getExportMetadata).toHaveBeenCalledTimes(3);
expect(downloadAs).toHaveBeenLastCalledWith({
format: "geojson",
geometry_column: "boundary",
@@ -1685,7 +2008,7 @@ describe("ExportActions GeoJSON clipboard", () => {
format: "geojson",
geometry_column: "boundary",
});
- expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ expect(getExportMetadata).toHaveBeenCalledTimes(3);
expect(downloadByURL).not.toHaveBeenCalled();
});
diff --git a/frontend/src/components/data-table/export-actions.tsx b/frontend/src/components/data-table/export-actions.tsx
index 3e55f962cb..926245965a 100644
--- a/frontend/src/components/data-table/export-actions.tsx
+++ b/frontend/src/components/data-table/export-actions.tsx
@@ -14,6 +14,7 @@ import {
TableIcon,
} from "lucide-react";
import React from "react";
+import useEvent from "react-use-event-hook";
import { downloadSizeLimitAtom } from "./download-policy/atoms";
import type {
DownloadAsArgs,
@@ -22,6 +23,7 @@ import type {
ExportMetadata,
GetExportMetadata,
} from "./schemas";
+import { useAsyncData } from "@/hooks/useAsyncData";
import { logNever } from "@/utils/assertNever";
import { cn } from "@/utils/cn";
import { copyToClipboard } from "@/utils/copy";
@@ -274,19 +276,130 @@ const failureTitleForAction = (action: ExportAction): string =>
const failureDescription = (error: unknown): string =>
typeof error === "string" ? error : prettyError(error);
+const OBSOLETE_GEOMETRY_MESSAGE =
+ "The selected geometry is no longer available. Choose another geometry to export.";
+
+type GeographicBlocker =
+ | "format-unavailable"
+ | "geometry-required"
+ | "choose-geometry"
+ | "obsolete-geometry"
+ | "missing-crs";
+
+/**
+ * Find the first reason that a geographic export cannot start with the
+ * selected geometry. The dialog uses it to disable a format, and the retry
+ * after a package installation uses it to decide whether to export.
+ */
+function geographicBlocker(
+ metadata: ExportMetadata,
+ format: GeographicFormat,
+ geometryColumn: string | null,
+): GeographicBlocker | null {
+ if (metadata.formats[format]?.available === false) {
+ return "format-unavailable";
+ }
+ const selected = metadata.geometry_columns.find(
+ (column) => column.name === geometryColumn,
+ );
+ if (geometryColumn !== null && !selected) {
+ return "obsolete-geometry";
+ }
+ if (metadata.geometry_columns.length === 0) {
+ return format === "geojson" ? "geometry-required" : null;
+ }
+ if (!selected) {
+ return "choose-geometry";
+ }
+ if (format === "geojson" && !selected.crs) {
+ return "missing-crs";
+ }
+ return null;
+}
+
+const geometryDisplayName = (name: string | null): string | null =>
+ name === "" ? "(unnamed geometry)" : name;
+
+function geographicBlockerReason(
+ blocker: GeographicBlocker | null,
+ formatLabel: string,
+ geometryColumn: string | null,
+): string | null {
+ switch (blocker) {
+ case "choose-geometry":
+ return `Choose a primary geometry column to export ${formatLabel}.`;
+ case "obsolete-geometry":
+ return OBSOLETE_GEOMETRY_MESSAGE;
+ case "missing-crs":
+ return `Geometry column ${geometryDisplayName(geometryColumn)} needs a CRS for GeoJSON export. Declare its source CRS in Python before exporting.`;
+ case "format-unavailable":
+ case "geometry-required":
+ case null:
+ return null;
+ default:
+ logNever(blocker);
+ return null;
+ }
+}
+
+/**
+ * One open dialog for one table source. Closing the dialog or replacing the
+ * source creates a new context, and work started under an older context must
+ * not change the dialog or start an export.
+ */
+interface ExportContext {
+ source: unknown;
+}
+
export const ExportActions: React.FC<ExportActionProps> = (props) => {
const { getExportMetadata, metadataSource } = props;
const [exportDialogOpen, setExportDialogOpen] = React.useState(false);
const [failure, setFailure] = React.useState<ExportFailure | null>(null);
const [settings, setSettings] =
React.useState<ExportSettings>(DEFAULT_SETTINGS);
- const [metadata, setMetadata] = React.useState<ExportMetadata | null>(null);
- const [metadataLoading, setMetadataLoading] = React.useState(false);
- const [metadataError, setMetadataError] = React.useState<string | null>(null);
- const [metadataRefresh, setMetadataRefresh] = React.useState(0);
+ const [exportContext, setExportContext] = React.useState<ExportContext>(
+ () => ({ source: metadataSource }),
+ );
+ if (exportContext.source !== metadataSource) {
+ setExportContext({ source: metadataSource });
+ }
+ const mounted = React.useRef(false);
+ const isCurrentContext = useEvent(
+ (context: ExportContext) => mounted.current && context === exportContext,
+ );
+ const metadataRequest = useAsyncData(async () => {
+ if (!exportDialogOpen || !getExportMetadata) {
+ return null;
+ }
+ return { context: exportContext, metadata: await getExportMetadata({}) };
+ }, [exportDialogOpen, getExportMetadata, exportContext]);
+ // The hook keeps earlier data while it reloads and after a failure. Only a
+ // completed response for the current context can enable an export.
+ const metadata =
+ metadataRequest.status === "success" &&
+ metadataRequest.data?.context === exportContext
+ ? metadataRequest.data.metadata
+ : null;
+ const metadataError =
+ metadataRequest.status === "error"
+ ? failureDescription(metadataRequest.error)
+ : null;
const [geometryColumn, setGeometryColumn] = React.useState<string | null>(
null,
);
+ const [reconciledMetadata, setReconciledMetadata] =
+ React.useState<ExportMetadata | null>(null);
+ if (metadata !== reconciledMetadata) {
+ setReconciledMetadata(metadata);
+ if (metadata) {
+ setGeometryColumn((current) =>
+ current !== null &&
+ metadata.geometry_columns.some((column) => column.name === current)
+ ? current
+ : metadata.default_geometry_column,
+ );
+ }
+ }
const hasGeometry = !!metadata?.geometry_columns.length;
const [expandedFormat, setExpandedFormat] = React.useState<
ConfigurableFormat | GeographicFormat | null
@@ -294,43 +407,17 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
const triggerRef = React.useRef<HTMLButtonElement>(null);
const dialogRef = React.useRef<HTMLDivElement>(null);
const latestActionId = React.useRef(0);
+ React.useLayoutEffect(() => {
+ mounted.current = true;
+ return () => {
+ mounted.current = false;
+ latestActionId.current += 1;
+ };
+ }, []);
React.useEffect(() => {
latestActionId.current += 1;
+ setFailure(null);
}, [metadataSource]);
- React.useEffect(() => {
- if (!exportDialogOpen || !getExportMetadata) {
- return;
- }
- let active = true;
- setMetadata(null);
- setMetadataLoading(true);
- setMetadataError(null);
- void getExportMetadata({}).then(
- (result) => {
- if (!active) {
- return;
- }
- setMetadata(result);
- setGeometryColumn((current) =>
- current !== null &&
- result.geometry_columns.some((column) => column.name === current)
- ? current
- : result.default_geometry_column,
- );
- setMetadataLoading(false);
- },
- (error: unknown) => {
- if (!active) {
- return;
- }
- setMetadataError(failureDescription(error));
- setMetadataLoading(false);
- },
- );
- return () => {
- active = false;
- };
- }, [exportDialogOpen, getExportMetadata, metadataSource, metadataRefresh]);
const policy = useAtomValue(downloadSizeLimitAtom);
const overLimit = !!(
policy &&
@@ -368,9 +455,8 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
setFailure(null);
setExpandedFormat(null);
setSettings(DEFAULT_SETTINGS);
- setMetadata(null);
- setMetadataError(null);
setGeometryColumn(null);
+ setExportContext({ source: metadataSource });
}
};
@@ -553,57 +639,67 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
}
};
- const retryAfterInstall = async (action: ExportAction) => {
- if (isGeographic(action.format) && getExportMetadata) {
+ // Package installation can finish after the dialog closes or the source
+ // changes, so the retry carries the context in which installation began.
+ const retryAfterInstall = useEvent(
+ async (action: ExportAction, context: ExportContext) => {
+ if (!isCurrentContext(context)) {
+ return;
+ }
+ if (!isGeographic(action.format) || !getExportMetadata) {
+ retryAction(action);
+ return;
+ }
const actionId = latestActionId.current;
+ const isCurrent = () =>
+ isCurrentContext(context) && actionId === latestActionId.current;
let result: ExportMetadata;
try {
result = await getExportMetadata({});
- } catch (error) {
- if (actionId === latestActionId.current) {
- setMetadataError(failureDescription(error));
+ } catch {
+ if (isCurrent()) {
+ metadataRequest.refetch();
}
return;
}
- if (actionId !== latestActionId.current) {
+ if (!isCurrent()) {
return;
}
- setMetadata(result);
- setMetadataError(null);
- if (result.formats[action.format]?.available === false) {
- setFailure(null);
- return;
- }
- if (
- geometryColumn !== null &&
- !result.geometry_columns.some(
- (column) => column.name === geometryColumn,
- )
- ) {
- setGeometryColumn(result.default_geometry_column);
- setActionFailure(actionId, {
- kind: "error",
- title: failureTitleForAction(action),
- description:
- "The selected geometry is no longer available. Choose another geometry to export.",
- action,
- });
- return;
- }
- if (result.geometry_columns.length > 0 && geometryColumn === null) {
- return;
- }
- if (
- action.format === "geojson" &&
- !result.geometry_columns.find(
- (column) => column.name === geometryColumn,
- )?.crs
- ) {
- setFailure(null);
- return;
+ // Validation reads its own response, so a display refresh cannot
+ // start an export that this check rejected.
+ metadataRequest.refetch();
+ const blocker = geographicBlocker(result, action.format, geometryColumn);
+ switch (blocker) {
+ case null:
+ retryAction(action);
+ return;
+ case "obsolete-geometry":
+ setGeometryColumn(result.default_geometry_column);
+ setActionFailure(actionId, {
+ kind: "error",
+ title: failureTitleForAction(action),
+ description: OBSOLETE_GEOMETRY_MESSAGE,
+ action,
+ });
+ return;
+ case "choose-geometry":
+ return;
+ case "format-unavailable":
+ case "geometry-required":
+ case "missing-crs":
+ setFailure(null);
+ return;
+ default:
+ logNever(blocker);
}
- }
- retryAction(action);
+ },
+ );
+
+ const createInstallRetry = (action: ExportAction) => {
+ const context = exportContext;
+ return () => {
+ void retryAfterInstall(action, context);
+ };
};
const toggleExpanded = (format: ExportFormat) => {
@@ -670,9 +766,7 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
failure?.action.format === option.format ? failure : null;
const summary =
isGeographic(option.format) && hasGeometry
- ? geometryColumn === ""
- ? "(unnamed geometry)"
- : (geometryColumn ?? "Choose geometry")
+ ? (geometryDisplayName(geometryColumn) ?? "Choose geometry")
: settingsSummary(settings, option.format);
const selectedGeometryIndex =
metadata?.geometry_columns.findIndex(
@@ -681,20 +775,14 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
const eligibility = isGeographic(option.format)
? metadata?.formats[option.format]
: undefined;
- const selectedGeometry =
- metadata?.geometry_columns[selectedGeometryIndex];
- const missingCrs =
- option.format === "geojson" &&
- selectedGeometry !== undefined &&
- !selectedGeometry.crs;
+ const blocker =
+ isGeographic(option.format) && metadata
+ ? geographicBlocker(metadata, option.format, geometryColumn)
+ : null;
const geometryReason = metadataError
? `Could not load geometry options: ${metadataError}`
: (eligibility?.reason ??
- (hasGeometry && geometryColumn === null
- ? `Choose a primary geometry column to export ${optionLabel}.`
- : missingCrs
- ? `Geometry column ${geometryColumn === "" ? "(unnamed geometry)" : geometryColumn} needs a CRS for GeoJSON export. Declare its source CRS in Python before exporting.`
- : null));
+ geographicBlockerReason(blocker, optionLabel, geometryColumn));
const missingPackageFailure =
rowFailure?.kind === "missing-packages" ? rowFailure : null;
const missingPackages =
@@ -704,13 +792,7 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
: []);
const geographicUnavailable =
isGeographic(option.format) &&
- ((!!getExportMetadata && !metadata) ||
- metadataLoading ||
- !!metadataError ||
- eligibility?.available === false ||
- missingCrs ||
- (!!metadata?.geometry_columns.length &&
- geometryColumn === null));
+ ((!!getExportMetadata && !metadata) || blocker !== null);
const copyUnavailable =
isGeographic(option.format) &&
(geographicUnavailable || missingPackages.length > 0);
@@ -888,9 +970,7 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
type="button"
size="xs"
variant="outline"
- onClick={() =>
- setMetadataRefresh((current) => current + 1)
- }
+ onClick={metadataRequest.refetch}
>
Retry
</Button>
@@ -906,17 +986,13 @@ export const ExportActions: React.FC<ExportActionProps> = (props) => {
missingPackageFailure?.description ??
eligibility?.reason
}
- onInstall={() => {
- void retryAfterInstall(
- missingPackageFailure?.action ?? {
- destination: "download",
- format:
- option.format === "geojson"
- ? "geojson"
- : "parquet",
- },
- );
- }}
+ onInstall={createInstallRetry(
+ missingPackageFailure?.action ?? {
+ destination: "download",
+ format:
+ option.format === "geojson" ? "geojson" : "parquet",
+ },
+ )}
className="items-start"
/>
</div>
diff --git a/frontend/src/components/data-table/schemas.ts b/frontend/src/components/data-table/schemas.ts
index 1e41b0703c..4cb8b9d263 100644
--- a/frontend/src/components/data-table/schemas.ts
+++ b/frontend/src/components/data-table/schemas.ts
@@ -3,42 +3,6 @@
import z from "zod";
import { rpc } from "@/plugins/core/rpc";
-export type DownloadFormat = "csv" | "json" | "parquet" | "tsv" | "geojson";
-
-/**
- * Per-request export options. Each field is optional. A missing field keeps
- * the widget's default for that setting.
- */
-export interface DownloadAsOptions {
- separator?: string;
- encoding?: string;
- ensure_ascii?: boolean;
-}
-
-export interface DownloadAsRequest {
- format: DownloadFormat;
- options?: DownloadAsOptions;
- geometry_column?: string | null;
-}
-
-export type DownloadAsArgs = (req: DownloadAsRequest) => Promise<{
- url: string;
- filename: string;
- error?: string | null;
- missing_packages?: string[] | null;
- code?:
- | "geometry_required"
- | "invalid_geometry"
- | "invalid_metadata"
- | "unsupported_representation"
- | "unsupported_version"
- | "missing_packages"
- | "missing_crs"
- | "conversion_failed"
- | null;
- column?: string | null;
-}>;
-
const DownloadRequestSchema = z.object({
geometry_column: z.string().nullish(),
options: z
@@ -88,6 +52,20 @@ export const DownloadGeoJSONSchema = rpc
}),
);
+export type DownloadAsRequest =
+ | z.infer<typeof DownloadAsSchema.input>
+ | z.infer<typeof DownloadGeoJSONSchema.input>;
+
+type DownloadResponse =
+ | z.infer<typeof DownloadAsSchema.output>
+ | z.infer<typeof DownloadGeoJSONSchema.output>;
+
+export type DownloadFormat = DownloadAsRequest["format"];
+export type DownloadAsOptions = NonNullable<DownloadAsRequest["options"]>;
+export type DownloadAsArgs = (
+ req: DownloadAsRequest,
+) => Promise<DownloadResponse>;
+
export type DownloadAsFunction = (
req: z.infer<typeof DownloadAsSchema.input>,
) => Promise<z.infer<typeof DownloadAsSchema.output>>;
diff --git a/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx b/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
index 598dfaf669..c093c0c682 100644
--- a/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
+++ b/frontend/src/plugins/impl/__tests__/DataTablePlugin.test.tsx
@@ -136,6 +136,90 @@ describe("LoadingDataTableComponent", () => {
expect(downloadGeoJSON).toHaveBeenCalledTimes(1);
});
+ it("refreshes export metadata when a dataframe transform changes the data", async () => {
+ const host = document.createElement("div");
+ const metadataFor = (name: string) => ({
+ geometry_columns: [{ name, encoding: "objects", crs: "EPSG:4326" }],
+ primary_geometry_column: name,
+ default_geometry_column: name,
+ formats: {
+ parquet: { available: true, reason: null, missing_packages: [] },
+ },
+ });
+ const getExportMetadata = vi
+ .fn()
+ .mockResolvedValueOnce(metadataFor("before"))
+ .mockResolvedValueOnce(metadataFor("after"));
+ const view = (data: string) => (
+ <Provider store={store}>
+ <TooltipProvider>
+ <LoadingDataTableComponent
+ label={null}
+ totalRows={1}
+ pagination={false}
+ pageSize={10}
+ selection={null}
+ showDownload={true}
+ showFilters={false}
+ showColumnSummaries={false}
+ showDataTypes={false}
+ showPageSizeSelector={false}
+ showColumnExplorer={false}
+ showRowExplorer={false}
+ showChartBuilder={false}
+ rowHeaders={[]}
+ fieldTypes={undefined}
+ totalColumns={1}
+ maxColumns="all"
+ hasStableRowId={false}
+ lazy={false}
+ host={host}
+ showSearch={false}
+ value={[]}
+ setValue={vi.fn()}
+ data={data}
+ search={vi.fn().mockResolvedValue({
+ data,
+ total_rows: 1,
+ cell_styles: null,
+ cell_hover_texts: null,
+ })}
+ download_as={vi.fn()}
+ download_geojson={vi.fn()}
+ get_export_metadata={getExportMetadata}
+ get_column_summaries={vi.fn()}
+ get_data_url={vi.fn() as GetDataUrl}
+ get_row_ids={vi.fn() as GetRowIds}
+ get_size_bytes={vi.fn().mockResolvedValue({ size_bytes: null })}
+ />
+ </TooltipProvider>
+ </Provider>
+ );
+ const { rerender } = render(
+ view(JSON.stringify([{ before: "POINT (1 2)" }])),
+ );
+ await waitFor(() =>
+ expect(
+ screen.getByRole("columnheader", { name: "before" }),
+ ).toBeInTheDocument(),
+ );
+ fireEvent.click(screen.getByTestId("export-button"));
+ await waitFor(() =>
+ expect(screen.getByTestId("export-summary-parquet")).toHaveTextContent(
+ "before",
+ ),
+ );
+
+ rerender(view(JSON.stringify([{ after: "POINT (3 4)" }])));
+
+ await waitFor(() =>
+ expect(screen.getByTestId("export-summary-parquet")).toHaveTextContent(
+ "after",
+ ),
+ );
+ expect(getExportMetadata).toHaveBeenCalledTimes(2);
+ });
+
it("does not log duplicate keys when a pivot becomes empty", async () => {
const host = document.createElement("div");
const consoleError = vi
diff --git a/marimo/_plugins/ui/_impl/tables/geometry.py b/marimo/_plugins/ui/_impl/tables/geometry.py
index b3ac2689c5..f12361ef96 100644
--- a/marimo/_plugins/ui/_impl/tables/geometry.py
+++ b/marimo/_plugins/ui/_impl/tables/geometry.py
@@ -77,12 +77,47 @@ _DUCKDB_2D_TYPES = frozenset(
)
+def arrow_storage_field(field: pa.Field[Any]) -> pa.Field[Any]:
+ """Describe a registered geometry extension field as storage plus metadata.
+
+ A registered extension type carries its name and declaration in the field
+ type. An unregistered one carries them in the field metadata. Geometry
+ fields of both kinds become a storage-typed field with the declaration in
+ `ARROW:extension:name` and `ARROW:extension:metadata`. Other fields are
+ returned unchanged.
+
+ Args:
+ field (pa.Field[Any]): A field from an Arrow schema.
+ """
+ import pyarrow as pa
+
+ if not isinstance(field.type, pa.ExtensionType):
+ return field
+ extension = field.type
+ if not (
+ extension.extension_name.startswith(_GEOARROW_PREFIX)
+ or extension.extension_name in _WKB_EXTENSION_NAMES
+ ):
+ return field
+ metadata: dict[bytes | str, bytes | str] = {
+ key: value for key, value in (field.metadata or {}).items()
+ }
+ metadata[b"ARROW:extension:name"] = extension.extension_name.encode()
+ serialized = extension.__arrow_ext_serialize__()
+ if extension.extension_name == "ogc.wkb" and not serialized:
+ metadata.pop(b"ARROW:extension:metadata", None)
+ else:
+ metadata[b"ARROW:extension:metadata"] = serialized
+ return field.with_type(extension.storage_type).with_metadata(metadata)
+
+
def _pyarrow_geometry_columns(
native: pa.Table,
) -> dict[str, GeometryColumnInfo]:
- """Detect geometry columns from pyarrow field metadata only."""
+ """Detect geometry columns from pyarrow schema declarations only."""
infos: dict[str, GeometryColumnInfo] = {}
for field in native.schema:
+ field = arrow_storage_field(field)
metadata = field.metadata
if not metadata:
continue
diff --git a/marimo/_plugins/ui/_impl/tables/geometry_export.py b/marimo/_plugins/ui/_impl/tables/geometry_export.py
index 07fdf4cae1..bda047831d 100644
--- a/marimo/_plugins/ui/_impl/tables/geometry_export.py
+++ b/marimo/_plugins/ui/_impl/tables/geometry_export.py
@@ -13,7 +13,7 @@ from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._impl.tables.geometry import (
GeometryColumnInfo,
GeometryEncoding,
- find_geometry_columns,
+ arrow_storage_field,
)
from marimo._plugins.ui._impl.tables.narwhals_table import (
NarwhalsTableManager,
@@ -175,12 +175,55 @@ def _missing_package_message(
return f"This {source_label} table needs {package} to export {label}."
+def _geographic_prerequisite_error(
+ manager: NarwhalsTableManager[Any, Any],
+ format_name: Literal["parquet", "geojson"],
+) -> GeometryExportError | None:
+ """Check current packages for a supported geographic export source."""
+ label = "GeoParquet" if format_name == "parquet" else "GeoJSON"
+ if (
+ manager.data.implementation.is_duckdb()
+ and not DependencyManager.pyarrow.has()
+ ):
+ return GeometryExportError(
+ "missing_packages",
+ _missing_package_message(manager, "pyarrow"),
+ missing_packages=["pyarrow"],
+ )
+ if not DependencyManager.geopandas.has():
+ return GeometryExportError(
+ "missing_packages",
+ _missing_package_message(manager, "geopandas", label),
+ missing_packages=["geopandas"],
+ )
+ if format_name == "parquet":
+ if not DependencyManager.geopandas.has_at_version(
+ min_version=_MIN_GEOPANDAS_GEOPARQUET_VERSION, quiet=True
+ ):
+ return GeometryExportError(
+ "unsupported_version", _GEOPANDAS_UPGRADE_MESSAGE
+ )
+ if not DependencyManager.pyarrow.has():
+ return GeometryExportError(
+ "missing_packages",
+ _missing_package_message(manager, "pyarrow", "GeoParquet"),
+ missing_packages=["pyarrow"],
+ )
+ elif not DependencyManager.shapely.has_at_version(
+ min_version=_MIN_SHAPELY_GEOJSON_VERSION, quiet=True
+ ):
+ return GeometryExportError(
+ "unsupported_version", _SHAPELY_UPGRADE_MESSAGE
+ )
+ return None
+
+
def _describe_export_source(manager: TableManager[Any]) -> _ExportSource:
"""Keep parsed source declarations separate from the metadata response."""
if not isinstance(manager, NarwhalsTableManager):
return _ExportSource(ExportMetadata())
- info_by_name = _export_geometry_columns(manager)
+ info_by_name = manager._geometry_columns
if not info_by_name:
return _ExportSource(ExportMetadata())
@@ -188,7 +231,6 @@ def _describe_export_source(manager: TableManager[Any]) -> _ExportSource:
primary: str | None = None
columns: list[GeometryExportColumn] = []
source_reason: str | None = None
- source_missing_packages: list[str] = []
schema: pa.Schema | None = None
declarations: dict[str, ArrowGeometryDeclaration] = {}
first_rejection: GeometryExportError | None = None
@@ -206,16 +248,13 @@ def _describe_export_source(manager: TableManager[Any]) -> _ExportSource:
crs=crs.to_string() if crs is not None else None,
)
)
- is_supported_source = DependencyManager.geopandas.has()
+ is_supported_source = True
elif (
source.implementation.is_pyarrow() or source.implementation.is_duckdb()
):
if source.implementation.is_duckdb():
if DependencyManager.pyarrow.has():
schema = _duckdb_geometry_schema(manager)
- else:
- source_reason = _missing_package_message(manager, "pyarrow")
- source_missing_packages = ["pyarrow"]
else:
schema = source.to_native().schema
for name, info in info_by_name.items():
@@ -247,7 +286,7 @@ def _describe_export_source(manager: TableManager[Any]) -> _ExportSource:
if source_reason is None and rejection is not None:
source_reason = rejection[1]
first_rejection = GeometryExportError(*rejection, column=name)
- is_supported_source = DependencyManager.geopandas.has()
+ is_supported_source = True
else:
columns = [
GeometryExportColumn(name=name, encoding=info.encoding, crs=None)
@@ -256,61 +295,29 @@ def _describe_export_source(manager: TableManager[Any]) -> _ExportSource:
is_supported_source = False
formats: dict[str, ExportFormatEligibility] = {}
- if not is_supported_source:
- for format_name, label in (
- ("parquet", "GeoParquet"),
- ("geojson", "GeoJSON"),
- ):
- if (
- source.implementation.is_pandas()
- or source.implementation.is_pyarrow()
- or source.implementation.is_duckdb()
- ):
- reason = _missing_package_message(manager, "geopandas", label)
- missing_packages = ["geopandas"]
- else:
- reason = (
- f"{label} export is not supported for this table type."
- )
- missing_packages = []
- formats[format_name] = ExportFormatEligibility(
- available=False,
- reason=reason,
- missing_packages=missing_packages,
- )
- else:
- if DependencyManager.shapely.has_at_version(
- min_version=_MIN_SHAPELY_GEOJSON_VERSION, quiet=True
- ):
- formats["geojson"] = ExportFormatEligibility(available=True)
- else:
- formats["geojson"] = ExportFormatEligibility(
- available=False, reason=_SHAPELY_UPGRADE_MESSAGE
- )
- if not DependencyManager.geopandas.has_at_version(
- min_version=_MIN_GEOPANDAS_GEOPARQUET_VERSION, quiet=True
- ):
- formats["parquet"] = ExportFormatEligibility(
- available=False,
- reason=_GEOPANDAS_UPGRADE_MESSAGE,
- )
- elif not DependencyManager.pyarrow.has():
- formats["parquet"] = ExportFormatEligibility(
- available=False,
- reason=_missing_package_message(
- manager, "pyarrow", "GeoParquet"
- ),
- missing_packages=["pyarrow"],
+ for format_name in ("parquet", "geojson"):
+ failure: GeometryExportError | None
+ if not is_supported_source:
+ label = "GeoParquet" if format_name == "parquet" else "GeoJSON"
+ failure = GeometryExportError(
+ "unsupported_representation",
+ f"{label} export is not supported for this table type.",
)
else:
- formats["parquet"] = ExportFormatEligibility(available=True)
+ failure = _geographic_prerequisite_error(manager, format_name)
+ formats[format_name] = ExportFormatEligibility(
+ available=failure is None,
+ reason=str(failure) if failure is not None else None,
+ missing_packages=(failure.missing_packages or [])
+ if failure is not None
+ else [],
+ )
if source_reason is not None:
formats = {
name: ExportFormatEligibility(
available=False,
reason=source_reason,
- missing_packages=source_missing_packages,
)
for name in ("parquet", "geojson")
}
@@ -359,7 +366,7 @@ def _duckdb_geometry_schema(
reader.close()
fields = []
for arrow_field in schema:
- arrow_field = _arrow_storage_field(arrow_field)
+ arrow_field = arrow_storage_field(arrow_field)
info = manager._geometry_columns.get(arrow_field.name)
if info is not None and info.encoding == "wkb":
metadata = dict(arrow_field.metadata or {})
@@ -380,7 +387,7 @@ def _duckdb_geometry_schema(
def _arrow_geometry_declaration(
field: pa.Field[Any],
) -> ArrowGeometryDeclaration:
- field = _arrow_storage_field(field)
+ field = arrow_storage_field(field)
metadata = field.metadata
if not metadata:
return ArrowGeometryDeclaration()
@@ -482,51 +489,10 @@ def has_geometry_columns(manager: TableManager[Any]) -> bool:
manager (TableManager[Any]): Manager for the export source.
"""
return isinstance(manager, NarwhalsTableManager) and bool(
- _export_geometry_columns(manager)
+ manager._geometry_columns
)
-def _arrow_storage_field(field: pa.Field[Any]) -> pa.Field[Any]:
- import pyarrow as pa
-
- if not isinstance(field.type, pa.ExtensionType):
- return field
- extension = field.type
- if not (
- extension.extension_name.startswith("geoarrow.")
- or extension.extension_name == "ogc.wkb"
- ):
- return field
- metadata: dict[bytes | str, bytes | str] = {
- key: value for key, value in (field.metadata or {}).items()
- }
- metadata[b"ARROW:extension:name"] = extension.extension_name.encode()
- serialized = extension.__arrow_ext_serialize__()
- if extension.extension_name == "ogc.wkb" and not serialized:
- metadata.pop(b"ARROW:extension:metadata", None)
- else:
- metadata[b"ARROW:extension:metadata"] = serialized
- return field.with_type(extension.storage_type).with_metadata(metadata)
-
-
-def _export_geometry_columns(
- manager: NarwhalsTableManager[Any, Any],
-) -> dict[str, GeometryColumnInfo]:
- if not manager.data.implementation.is_pyarrow():
- return find_geometry_columns(manager.data)
-
- import pyarrow as pa
-
- schema = pa.schema(
- [
- _arrow_storage_field(field)
- for field in manager.data.to_native().schema
- ]
- )
- schema_view = pa.Table.from_batches([], schema=schema)
- return find_geometry_columns(nw.from_native(schema_view))
-
-
def _arrow_to_geodataframe(
table: pa.Table,
columns: list[GeometryExportColumn],
@@ -651,40 +617,9 @@ def _prepare_geographic_export(
"geometry_required",
f"Choose a {'primary ' if format_name == 'parquet' else ''}geometry column for {label} export.",
)
- if (
- manager.data.implementation.is_duckdb()
- and not DependencyManager.pyarrow.has()
- ):
- raise GeometryExportError(
- "missing_packages",
- _missing_package_message(manager, "pyarrow"),
- missing_packages=["pyarrow"],
- )
- if not DependencyManager.geopandas.has():
- raise GeometryExportError(
- "missing_packages",
- _missing_package_message(manager, "geopandas", label),
- missing_packages=["geopandas"],
- )
- if format_name == "parquet":
- if not DependencyManager.geopandas.has_at_version(
- min_version=_MIN_GEOPANDAS_GEOPARQUET_VERSION, quiet=True
- ):
- raise GeometryExportError(
- "unsupported_version", _GEOPANDAS_UPGRADE_MESSAGE
- )
- if not DependencyManager.pyarrow.has():
- raise GeometryExportError(
- "missing_packages",
- _missing_package_message(manager, "pyarrow", "GeoParquet"),
- missing_packages=["pyarrow"],
- )
- elif not DependencyManager.shapely.has_at_version(
- min_version=_MIN_SHAPELY_GEOJSON_VERSION, quiet=True
- ):
- raise GeometryExportError(
- "unsupported_version", _SHAPELY_UPGRADE_MESSAGE
- )
+ failure = _geographic_prerequisite_error(manager, format_name)
+ if failure is not None:
+ raise failure
if source.rejection is not None:
raise source.rejection
if (
@@ -1211,7 +1146,7 @@ def _validate_geoparquet(
or "crs" not in column
or name not in schema.names
or not pa.types.is_binary(
- _arrow_storage_field(schema.field(name)).type
+ arrow_storage_field(schema.field(name)).type
)
):
raise GeometryExportError(
@@ -1308,7 +1243,7 @@ def _prepare_arrow_export(
continue
index = export_table.schema.get_field_index(column)
- field = export_table.schema.field(index)
+ field = arrow_storage_field(export_table.schema.field(index))
metadata = dict(field.metadata or {})
metadata.pop(b"ARROW:extension:name", None)
metadata.pop(b"ARROW:extension:metadata", None)
@@ -1320,7 +1255,15 @@ def _prepare_arrow_export(
metadata=metadata or None,
)
- values = export_table.column(index)
+ values = pa.chunked_array(
+ [
+ chunk.storage
+ if isinstance(chunk, pa.ExtensionArray)
+ else chunk
+ for chunk in export_table.column(index).chunks
+ ],
+ type=field.type,
+ )
export_values: pa.Array[Any] | pa.ChunkedArray[Any]
if info.encoding == "wkb":
export_values = pa.array(
diff --git a/tests/_plugins/ui/_impl/tables/geometry_fixtures.py b/tests/_plugins/ui/_impl/tables/geometry_fixtures.py
index 30ab2f472b..9bfdbd8e4d 100644
--- a/tests/_plugins/ui/_impl/tables/geometry_fixtures.py
+++ b/tests/_plugins/ui/_impl/tables/geometry_fixtures.py
@@ -15,11 +15,14 @@ from __future__ import annotations
import datetime
import json
+from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
import pytest
if TYPE_CHECKING:
+ from collections.abc import Generator
+
import duckdb as duckdb_mod
import geopandas as gpd_mod
import ibis as ibis_mod
@@ -294,6 +297,61 @@ def arrow_invalid_wkb() -> pa_mod.Table:
)
+@contextmanager
+def registered_arrow_extension(
+ name: str,
+ storage_type: pa_mod.DataType,
+ serialized: bytes = b'{"crs":"EPSG:4326"}',
+) -> Generator[pa_mod.ExtensionType, None, None]:
+ """Register an Arrow extension type for the duration of a test.
+
+ Registered types carry their name in the field type instead of the field
+ metadata, as GeoArrow-aware readers produce them. Deserialization keeps
+ the serialized payload, so Arrow data that DuckDB or Parquet reproduce
+ retains its declaration.
+ """
+ import pyarrow as pa
+
+ class RegisteredExtension(pa.ExtensionType):
+ def __init__(
+ self, storage: pa.DataType, payload: bytes = serialized
+ ) -> None:
+ self.payload = payload
+ super().__init__(storage, name)
+
+ def __arrow_ext_serialize__(self) -> bytes:
+ return self.payload
+
+ @classmethod
+ def __arrow_ext_deserialize__(
+ cls, storage_type: pa.DataType, serialized: bytes
+ ) -> RegisteredExtension:
+ return cls(storage_type, serialized)
+
+ extension = RegisteredExtension(storage_type)
+ pa.register_extension_type(extension) # type: ignore[arg-type]
+ try:
+ yield extension
+ finally:
+ pa.unregister_extension_type(name)
+
+
+def registered_extension_column(
+ extension: pa_mod.ExtensionType, chunks: list[list[Any]]
+) -> pa_mod.ChunkedArray[Any]:
+ """Build an extension column with one chunk per value list."""
+ import pyarrow as pa
+
+ return pa.chunked_array(
+ [
+ pa.ExtensionArray.from_storage(
+ extension, pa.array(values, type=extension.storage_type)
+ )
+ for values in chunks
+ ]
+ )
+
+
# --- DuckDB spatial ----------------------------------------------------
diff --git a/tests/_plugins/ui/_impl/tables/test_geometry.py b/tests/_plugins/ui/_impl/tables/test_geometry.py
index 03c0d38301..88c22b2ad2 100644
--- a/tests/_plugins/ui/_impl/tables/test_geometry.py
+++ b/tests/_plugins/ui/_impl/tables/test_geometry.py
@@ -150,6 +150,46 @@ class TestArrowDetection:
assert find_geometry_columns(nw.from_native(table)) == {}
+ @pytest.mark.parametrize(
+ ("name", "encoding"),
+ [
+ ("geoarrow.wkb", "wkb"),
+ ("ogc.wkb", "wkb"),
+ ("geoarrow.wkt", "wkt"),
+ ],
+ )
+ def test_detects_registered_extension_types(
+ self, name: str, encoding: str
+ ) -> None:
+ import narwhals.stable.v2 as nw
+ import pyarrow as pa
+
+ storage_type = pa.string() if encoding == "wkt" else pa.binary()
+ with geo.registered_arrow_extension(name, storage_type) as extension:
+ table = pa.table(
+ {"geom": geo.registered_extension_column(extension, [[None]])}
+ )
+
+ assert find_geometry_columns(nw.from_native(table)) == {
+ "geom": GeometryColumnInfo(
+ encoding=encoding, # type: ignore[arg-type]
+ external_type=name,
+ )
+ }
+
+ def test_ignores_registered_non_geo_extension(self) -> None:
+ import narwhals.stable.v2 as nw
+ import pyarrow as pa
+
+ with geo.registered_arrow_extension(
+ "example.label", pa.string(), b""
+ ) as extension:
+ table = pa.table(
+ {"label": geo.registered_extension_column(extension, [["a"]])}
+ )
+
+ assert find_geometry_columns(nw.from_native(table)) == {}
+
def test_manager_field_type_uses_geometry_semantics(self) -> None:
manager = get_table_manager(geo.arrow_wkb_known_crs())
diff --git a/tests/_plugins/ui/_impl/test_export_metadata.py b/tests/_plugins/ui/_impl/test_export_metadata.py
index 54cbdde1a7..594a504449 100644
--- a/tests/_plugins/ui/_impl/test_export_metadata.py
+++ b/tests/_plugins/ui/_impl/test_export_metadata.py
@@ -170,6 +170,138 @@ def test_missing_geopandas_is_reported(widget: Any) -> None:
}
[email protected]("geopandas", "pyarrow")
+ "factory", [fixtures.gdf_multi_geometry, fixtures.arrow_multi_geometry]
+)
+ ("format_name", "package", "method", "code", "missing_packages"),
+ [
+ ("parquet", "geopandas", "has", "missing_packages", ["geopandas"]),
+ ("geojson", "geopandas", "has", "missing_packages", ["geopandas"]),
+ ("parquet", "pyarrow", "has", "missing_packages", ["pyarrow"]),
+ ("parquet", "geopandas", "has_at_version", "unsupported_version", []),
+ ("geojson", "shapely", "has_at_version", "unsupported_version", []),
+ ],
+)
+def test_metadata_and_download_share_prerequisites(
+ widget: Any,
+ factory: Any,
+ format_name: Literal["parquet", "geojson"],
+ package: str,
+ method: str,
+ code: str,
+ missing_packages: list[str],
+) -> None:
+ subject = widget(factory())
+ with patch.object(
+ getattr(DependencyManager, package), method, return_value=False
+ ):
+ eligibility = subject._get_export_metadata(EmptyArgs()).formats[
+ format_name
+ ]
+ response = subject._download_as(
+ DownloadAsArgs(format=format_name, geometry_column="geom_a")
+ )
+ assert asdict(eligibility) == {
+ "available": False,
+ "reason": response.error,
+ "missing_packages": missing_packages,
+ }
+ assert (response.url, response.code, response.missing_packages) == (
+ "",
+ code,
+ missing_packages or None,
+ )
+
+
[email protected]("geopandas", "pyarrow")
+ "factory", [fixtures.gdf_multi_geometry, fixtures.arrow_multi_geometry]
+)
[email protected]("format_name", ["parquet", "geojson"])
+def test_package_installation_changes_eligibility_and_download(
+ widget: Any, factory: Any, format_name: Literal["parquet", "geojson"]
+) -> None:
+ subject = widget(factory())
+ with patch.object(DependencyManager.geopandas, "has", return_value=False):
+ assert (
+ not subject._get_export_metadata(EmptyArgs())
+ .formats[format_name]
+ .available
+ )
+ assert (
+ subject._download_as(
+ DownloadAsArgs(format=format_name, geometry_column="geom_a")
+ ).code
+ == "missing_packages"
+ )
+ assert (
+ subject._get_export_metadata(EmptyArgs())
+ .formats[format_name]
+ .available
+ )
+ response = subject._download_as(
+ DownloadAsArgs(format=format_name, geometry_column="geom_a")
+ )
+ assert response.url
+ assert response.error is None
+
+
[email protected]("geopandas", "pyarrow")
+ "factory", [fixtures.gdf_multi_geometry, fixtures.arrow_multi_geometry]
+)
[email protected]("format_name", ["parquet", "geojson"])
+def test_invalid_choice_precedes_missing_package(
+ widget: Any, factory: Any, format_name: Literal["parquet", "geojson"]
+) -> None:
+ subject = widget(factory())
+ with patch.object(DependencyManager.geopandas, "has", return_value=False):
+ response = subject._download_as(
+ DownloadAsArgs(format=format_name, geometry_column="missing")
+ )
+ assert (response.code, response.column, response.missing_packages) == (
+ "invalid_geometry",
+ "missing",
+ None,
+ )
+ assert response.url == ""
+
+
[email protected]("geopandas", "pyarrow")
[email protected]("format_name", ["parquet", "geojson"])
+def test_invalid_declaration_and_missing_package_keep_error_precedence(
+ widget: Any, format_name: Literal["parquet", "geojson"]
+) -> None:
+ subject = widget(fixtures.arrow_malformed_metadata())
+ with patch.object(DependencyManager.geopandas, "has", return_value=False):
+ eligibility = subject._get_export_metadata(EmptyArgs()).formats[
+ format_name
+ ]
+ response = subject._download_as(
+ DownloadAsArgs(format=format_name, geometry_column="geom")
+ )
+ assert asdict(eligibility) == {
+ "available": False,
+ "reason": "Geometry column 'geom' has invalid GeoArrow metadata: expected a JSON object.",
+ "missing_packages": [],
+ }
+ assert (response.url, response.code, response.missing_packages) == (
+ "",
+ "missing_packages",
+ ["geopandas"],
+ )
+ response = subject._download_as(
+ DownloadAsArgs(format=format_name, geometry_column="geom")
+ )
+ assert (response.url, response.code, response.error) == (
+ "",
+ "invalid_metadata",
+ eligibility.reason,
+ )
+
+
@pytest.mark.requires("geopandas", "pyarrow")
def test_old_shapely_disables_only_geojson(widget: Any) -> None:
subject = widget(fixtures.gdf_multi_geometry())
diff --git a/tests/_plugins/ui/_impl/test_geojson_export.py b/tests/_plugins/ui/_impl/test_geojson_export.py
index 8e5caa1bb0..e588a96b45 100644
--- a/tests/_plugins/ui/_impl/test_geojson_export.py
+++ b/tests/_plugins/ui/_impl/test_geojson_export.py
@@ -1082,28 +1082,13 @@ def test_arrow_geojson_registered_primary_and_secondary_geometry(
import pyarrow as pa
from shapely import from_wkt
- name = f"geoarrow.{encoding}"
storage_type = pa.binary() if encoding == "wkb" else pa.string()
-
- class GeometryType(pa.ExtensionType):
- def __init__(self) -> None:
- super().__init__(storage_type, name)
-
- def __arrow_ext_serialize__(self) -> bytes:
- return b'{"crs":"EPSG:4326"}'
-
- @classmethod
- def __arrow_ext_deserialize__(
- cls, _storage_type: Any, _serialized: bytes
- ) -> Any:
- return cls()
-
- extension = GeometryType()
- pa.register_extension_type(extension) # type: ignore[arg-type]
- try:
+ with fixtures.registered_arrow_extension(
+ f"geoarrow.{encoding}", storage_type
+ ) as extension:
value = fixtures.WKB_POINT_1_2 if encoding == "wkb" else "POINT (1 2)"
- values = pa.ExtensionArray.from_storage(
- extension, pa.array([value, None], type=storage_type)
+ values = fixtures.registered_extension_column(
+ extension, [[value, None]]
)
source = fixtures.arrow_multi_geometry().append_column(
"registered", values
@@ -1122,8 +1107,6 @@ def test_arrow_geojson_registered_primary_and_secondary_geometry(
assert features[1]["geometry"] is None
assert features[1]["properties"][secondary] is None
assert source["registered"].to_pylist() == [value, None]
- finally:
- pa.unregister_extension_type(name)
@pytest.mark.requires("geopandas", "pyarrow")
@@ -1513,29 +1496,18 @@ def test_duckdb_closed_connection_is_structured_failure() -> None:
def test_duckdb_registered_wkb_preserves_crs(widget: Any) -> None:
import pyarrow as pa
- class GeometryType(pa.ExtensionType):
- def __init__(self, metadata: bytes = b"{}") -> None:
- self.metadata = metadata
- super().__init__(pa.binary(), "geoarrow.wkb")
-
- def __arrow_ext_serialize__(self) -> bytes:
- return self.metadata
-
- @classmethod
- def __arrow_ext_deserialize__(
- cls, storage: Any, metadata: bytes
- ) -> Any:
- return cls(metadata)
-
conn = fixtures.duckdb_crs_geometry_connection()
- pa.register_extension_type(GeometryType()) # type: ignore[arg-type]
try:
- document = _artifact(
- widget(
- conn.sql("SELECT 'POINT (1 2)'::GEOMETRY('OGC:CRS84') AS geom")
+ with fixtures.registered_arrow_extension(
+ "geoarrow.wkb", pa.binary(), b"{}"
+ ):
+ document = _artifact(
+ widget(
+ conn.sql(
+ "SELECT 'POINT (1 2)'::GEOMETRY('OGC:CRS84') AS geom"
+ )
+ )
)
- )
assert document["features"][0]["geometry"]["coordinates"] == [1, 2]
finally:
- pa.unregister_extension_type("geoarrow.wkb")
conn.close()
diff --git a/tests/_plugins/ui/_impl/test_geoparquet_export.py b/tests/_plugins/ui/_impl/test_geoparquet_export.py
index ca47e59450..4f906a8cc9 100644
--- a/tests/_plugins/ui/_impl/test_geoparquet_export.py
+++ b/tests/_plugins/ui/_impl/test_geoparquet_export.py
@@ -819,36 +819,18 @@ def test_arrow_geoparquet_registered_extension_storage(
import pyarrow as pa
import pyarrow.parquet as pq
- name = f"geoarrow.{encoding}"
storage_type = pa.binary() if encoding == "wkb" else pa.string()
-
- class GeometryType(pa.ExtensionType):
- def __init__(self) -> None:
- super().__init__(storage_type, name)
-
- def __arrow_ext_serialize__(self) -> bytes:
- return b'{"crs":"EPSG:4326"}'
-
- @classmethod
- def __arrow_ext_deserialize__(
- cls, _storage_type: Any, _serialized: bytes
- ) -> Any:
- return cls()
-
- extension = GeometryType()
- pa.register_extension_type(extension) # type: ignore[arg-type]
- try:
+ with fixtures.registered_arrow_extension(
+ f"geoarrow.{encoding}", storage_type
+ ) as extension:
value = fixtures.WKB_POINT_1_2 if encoding == "wkb" else "POINT (1 2)"
- chunks = [
- pa.ExtensionArray.from_storage(
- extension, pa.array([value], type=storage_type)
- ),
- pa.ExtensionArray.from_storage(
- extension, pa.array([None], type=storage_type)
- ),
- ]
source = pa.Table.from_arrays(
- [pa.array(["point", "null"]), pa.chunked_array(chunks)],
+ [
+ pa.array(["point", "null"]),
+ fixtures.registered_extension_column(
+ extension, [[value], [None]]
+ ),
+ ],
names=["name", "geom"],
)
subject = widget(source)
@@ -866,8 +848,6 @@ def test_arrow_geoparquet_registered_extension_storage(
assert frame.crs.to_epsg() == 4326
assert source["geom"].num_chunks == 2
assert source["geom"].to_pylist() == [value, None]
- finally:
- pa.unregister_extension_type(name)
@pytest.mark.requires("geopandas", "pyarrow")
@@ -1099,25 +1079,11 @@ def test_registered_ogc_wkb_metadata(widget: Any, payload: bytes) -> None:
from marimo._plugins.ui._impl.table import DownloadGeoJSONArgs
- class OGCType(pa.ExtensionType):
- def __init__(self) -> None:
- super().__init__(pa.binary(), "ogc.wkb")
-
- def __arrow_ext_serialize__(self) -> bytes:
- return payload
-
- @classmethod
- def __arrow_ext_deserialize__(
- cls, _storage: Any, _serialized: bytes
- ) -> Any:
- return cls()
-
- extension = OGCType()
- pa.register_extension_type(extension) # type: ignore[arg-type]
- try:
- values = pa.ExtensionArray.from_storage(
- extension,
- pa.array([fixtures.WKB_POINT_1_2, None], type=pa.binary()),
+ with fixtures.registered_arrow_extension(
+ "ogc.wkb", pa.binary(), payload
+ ) as extension:
+ values = fixtures.registered_extension_column(
+ extension, [[fixtures.WKB_POINT_1_2, None]]
)
source = pa.Table.from_arrays([values], names=["geom"])
subject = widget(source)
@@ -1139,8 +1105,6 @@ def test_registered_ogc_wkb_metadata(widget: Any, payload: bytes) -> None:
)
assert (response.code, response.url) == ("missing_crs", "")
assert source["geom"].to_pylist() == [fixtures.WKB_POINT_1_2, None]
- finally:
- pa.unregister_extension_type("ogc.wkb")
@pytest.mark.requires("geopandas", "pyarrow")
diff --git a/tests/_plugins/ui/_impl/test_table.py b/tests/_plugins/ui/_impl/test_table.py
index 652cef5455..7f2ef95e33 100644
--- a/tests/_plugins/ui/_impl/test_table.py
+++ b/tests/_plugins/ui/_impl/test_table.py
@@ -1831,6 +1831,74 @@ def test_download_arrow_wkb_as_complete_wkt(format_type: str) -> None:
assert source.schema.field("geometry").metadata == metadata
[email protected]("pyarrow", "shapely")
[email protected]("format_type", TEXT_DOWNLOAD_FORMATS)
[email protected]("encoding", ["wkb", "wkt"])
+def test_download_registered_arrow_geometry_as_complete_text(
+ format_type: str, encoding: str
+) -> None:
+ import pyarrow as pa
+ from shapely import from_wkt
+
+ from tests._plugins.ui._impl.tables import geometry_fixtures as fixtures
+
+ long_geometry = from_wkt(LONG_GEOMETRY_WKT)
+ if encoding == "wkb":
+ storage_type = pa.binary()
+ values = [long_geometry.wkb, from_wkt(POINT_GEOMETRY_WKT).wkb, None]
+ expected_long = long_geometry.wkt
+ else:
+ storage_type = pa.string()
+ values = [LONG_GEOMETRY_WKT, POINT_GEOMETRY_WKT, None]
+ expected_long = LONG_GEOMETRY_WKT
+
+ with fixtures.registered_arrow_extension(
+ f"geoarrow.{encoding}", storage_type
+ ) as extension:
+ geometry = fixtures.registered_extension_column(
+ extension, [values[:2], values[2:]]
+ )
+ source = pa.table(
+ {"row_label": ["long", "point", "null"], "geometry": geometry}
+ )
+
+ rows = _download_text_rows(ui.table(source), format_type)
+
+ _assert_complete_geometry_rows(rows, format_type, expected_long)
+ assert source.schema.field("geometry").type == extension
+ assert source.column("geometry").num_chunks == 2
+ assert source.column("geometry").to_pylist() == values
+
+
+def test_download_registered_arrow_wkb_without_shapely_as_hex() -> None:
+ import pyarrow as pa
+
+ from tests._plugins.ui._impl.tables import geometry_fixtures as fixtures
+
+ wkb = fixtures.WKB_POINT_1_2
+ with fixtures.registered_arrow_extension(
+ "geoarrow.wkb", pa.binary()
+ ) as extension:
+ source = pa.table(
+ {
+ "row_label": ["point", "null"],
+ "geometry": fixtures.registered_extension_column(
+ extension, [[wkb, None]]
+ ),
+ }
+ )
+ with patch.object(
+ DependencyManager.shapely, "has", return_value=False
+ ):
+ rows = _download_text_rows(ui.table(source), "csv")
+
+ assert rows == [
+ {"row_label": "point", "geometry": wkb.hex()},
+ {"row_label": "null", "geometry": ""},
+ ]
+
+
@pytest.mark.requires("duckdb", "pyarrow")
@pytest.mark.parametrize("format_type", TEXT_DOWNLOAD_FORMATS)
def test_download_duckdb_geometry_as_complete_text(