Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

feat(export-data): add GeoJSON, GeoParquet, Shapefile, GeoPackage, KML, and KMZ (#3811)

Igor Dykhtaebcf4e866ab7
diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.ts
index 6af6fed98..4c98a55df 100644
--- a/src/actions/src/action-types.ts
+++ b/src/actions/src/action-types.ts
@@ -115,6 +115,8 @@ export const ActionTypes = {
SET_MAP_INFO: `${ACTION_PREFIX}SET_MAP_INFO`,
SORT_TABLE_COLUMN: `${ACTION_PREFIX}SORT_TABLE_COLUMN`,
PIN_TABLE_COLUMN: `${ACTION_PREFIX}PIN_TABLE_COLUMN`,
+ TOGGLE_TABLE_COLUMN_HIDDEN: `${ACTION_PREFIX}TOGGLE_TABLE_COLUMN_HIDDEN`,
+ SHOW_ALL_TABLE_COLUMNS: `${ACTION_PREFIX}SHOW_ALL_TABLE_COLUMNS`,
COPY_TABLE_COLUMN: `${ACTION_PREFIX}COPY_TABLE_COLUMN`,
SET_COLUMN_DISPLAY_FORMAT: `${ACTION_PREFIX}SET_COLUMN_DISPLAY_FORMAT`,
LOAD_COLUMN_STATS: `${ACTION_PREFIX}LOAD_COLUMN_STATS`,
diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.ts
index c17cdfccb..ec68c1d64 100644
--- a/src/actions/src/vis-state-actions.ts
+++ b/src/actions/src/vis-state-actions.ts
@@ -1251,6 +1251,49 @@ export function pinTableColumn(
};
}
+export type ToggleTableColumnHiddenUpdaterAction = {
+ dataId: string;
+ column: string;
+};
+/**
+ * Hide or show a dataset column in the attribute table and in data export.
+ * @param dataId
+ * @param column
+ * @returns action
+ * @public
+ */
+export function toggleTableColumnHidden(
+ dataId: string,
+ column: string
+): Merge<
+ ToggleTableColumnHiddenUpdaterAction,
+ {type: typeof ActionTypes.TOGGLE_TABLE_COLUMN_HIDDEN}
+> {
+ return {
+ type: ActionTypes.TOGGLE_TABLE_COLUMN_HIDDEN,
+ dataId,
+ column
+ };
+}
+
+export type ShowAllTableColumnsUpdaterAction = {
+ dataId: string;
+};
+/**
+ * Show every attribute-table column that was hidden.
+ * @param dataId
+ * @returns action
+ * @public
+ */
+export function showAllTableColumns(
+ dataId: string
+): Merge<ShowAllTableColumnsUpdaterAction, {type: typeof ActionTypes.SHOW_ALL_TABLE_COLUMNS}> {
+ return {
+ type: ActionTypes.SHOW_ALL_TABLE_COLUMNS,
+ dataId
+ };
+}
+
export type CopyTableColumnUpdaterAction = {
dataId: string;
column: string;
diff --git a/src/components/src/common/data-table/header-cell.tsx b/src/components/src/common/data-table/header-cell.tsx
index 196721a79..0b4e14c07 100644
--- a/src/components/src/common/data-table/header-cell.tsx
+++ b/src/components/src/common/data-table/header-cell.tsx
@@ -200,6 +200,7 @@ function HeaderCellFactory(
sortColumn,
sortTableColumn,
pinTableColumn,
+ hideTableColumn,
copyTableColumn,
setColumnDisplayFormat,
hasStats,
@@ -254,6 +255,9 @@ function HeaderCellFactory(
const onPin = useCallback(() => {
if (!isGhost) pinTableColumn(column);
}, [pinTableColumn, isGhost, column]);
+ const onHide = useCallback(() => {
+ if (!isGhost) hideTableColumn?.(column);
+ }, [hideTableColumn, isGhost, column]);
const onCopy = useCallback(() => {
if (!isGhost) copyTableColumn(column);
}, [copyTableColumn, isGhost, column]);
@@ -375,6 +379,7 @@ function HeaderCellFactory(
toggleMoreOptions={toggleMoreOptions}
sortTableColumn={sortTableColumn ? mode => sortTableColumn(column, mode) : undefined}
pinTableColumn={onPin}
+ hideTableColumn={hideTableColumn ? onHide : undefined}
copyTableColumn={onCopy}
setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}
renameTableColumn={renameTableColumn}
diff --git a/src/components/src/common/data-table/index.tsx b/src/components/src/common/data-table/index.tsx
index 905c1c5cf..d7873d767 100644
--- a/src/components/src/common/data-table/index.tsx
+++ b/src/components/src/common/data-table/index.tsx
@@ -259,14 +259,14 @@ interface GetRowCellProps {
/*
* This is an accessor method used to generalize getting a cell from a data row
*/
-const defaultGetRowCell = (
+export const defaultGetRowCell = (
{dataContainer, columns, column, colMeta, rowIndex, sortOrder}: GetRowCellProps,
formatter
) => {
const rowIdx = sortOrder && sortOrder.length ? get(sortOrder, rowIndex) : rowIndex;
- const {type} = colMeta[column];
+ const {type, colIdx} = colMeta[column];
- const value = dataContainer?.valueAt(rowIdx, columns.indexOf(column));
+ const value = dataContainer?.valueAt(rowIdx, colIdx ?? columns.indexOf(column));
if (value === null || value === undefined || value === '') {
return '';
}
@@ -454,6 +454,7 @@ export interface DataTableProps {
sortColumn: SortColumn;
sortTableColumn: (column: string, mode?: string) => void;
pinTableColumn: (column: string) => void;
+ hideTableColumn?: (column: string) => void;
setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;
copyTableColumn: (column: string) => void;
renameTableColumn?: (column: string, newName: string) => void;
diff --git a/src/components/src/common/data-table/option-dropdown.tsx b/src/components/src/common/data-table/option-dropdown.tsx
index 5d2bfa06f..b7d43216d 100644
--- a/src/components/src/common/data-table/option-dropdown.tsx
+++ b/src/components/src/common/data-table/option-dropdown.tsx
@@ -23,7 +23,8 @@ import {
Hash,
Edit,
Trash,
- FilterFunnel
+ FilterFunnel,
+ EyeUnseen
} from '../icons';
const ListItem = ({value}) => (
@@ -93,10 +94,10 @@ const StyledDeleteConfirm = styled.div`
const StyledPopover = styled.div`
width: 184px;
- height: 160px;
+ height: 200px;
z-index: 101;
.list-selector {
- max-height: 160px;
+ max-height: 200px;
}
.hover:after {
content: '\\2713';
@@ -158,6 +159,7 @@ export interface OptionDropdownProps {
toggleMoreOptions: (column: string) => void;
sortTableColumn?: (sort: string) => void;
pinTableColumn: () => void;
+ hideTableColumn?: () => void;
copyTableColumn: () => void;
setDisplayFormat?: (displayFormat: any) => void;
sortMode?: string;
@@ -177,6 +179,7 @@ const OptionDropdown = (props: OptionDropdownProps) => {
toggleMoreOptions,
sortTableColumn,
pinTableColumn,
+ hideTableColumn,
copyTableColumn,
setDisplayFormat,
deleteTableColumn,
@@ -206,6 +209,9 @@ const OptionDropdown = (props: OptionDropdownProps) => {
case TABLE_OPTION.COPY:
copyTableColumn();
break;
+ case TABLE_OPTION.HIDE:
+ hideTableColumn?.();
+ break;
case TABLE_OPTION.FORMAT_COLUMN:
setShowFormatter(true);
return;
@@ -228,6 +234,7 @@ const OptionDropdown = (props: OptionDropdownProps) => {
column,
sortTableColumn,
pinTableColumn,
+ hideTableColumn,
copyTableColumn,
toggleMoreOptions,
onStartRename,
@@ -244,7 +251,8 @@ const OptionDropdown = (props: OptionDropdownProps) => {
Hash,
Edit,
Trash,
- FilterFunnel
+ FilterFunnel,
+ EyeUnseen
};
const formatLabels = getFieldFormatLabels(colMeta[column].type);
diff --git a/src/components/src/modal-container.spec.ts b/src/components/src/modal-container.spec.ts
new file mode 100644
index 000000000..bd131c9cb
--- /dev/null
+++ b/src/components/src/modal-container.spec.ts
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {EXPORT_DATA_ID, EXPORT_MAP_ID} from '@kepler.gl/constants';
+
+import {exportDataFromModal} from './modal-container';
+
+function deferred() {
+ let resolve!: () => void;
+ let reject!: (error: Error) => void;
+ const promise = new Promise<void>((resolvePromise, rejectPromise) => {
+ resolve = resolvePromise;
+ reject = rejectPromise;
+ });
+ return {promise, resolve, reject};
+}
+
+describe('exportDataFromModal', () => {
+ test('closes the export modal after a successful export', async () => {
+ const closeModal = jest.fn();
+ await exportDataFromModal({
+ exportData: () => Promise.resolve(),
+ getCurrentModal: () => EXPORT_DATA_ID,
+ closeModal,
+ addNotification: jest.fn()
+ });
+ expect(closeModal).toHaveBeenCalledTimes(1);
+ });
+
+ test('does not close a newer modal after export completes', async () => {
+ const exportTask = deferred();
+ const closeModal = jest.fn();
+ let currentModal = EXPORT_DATA_ID;
+ const completion = exportDataFromModal({
+ exportData: () => exportTask.promise,
+ getCurrentModal: () => currentModal,
+ closeModal,
+ addNotification: jest.fn()
+ });
+
+ currentModal = EXPORT_MAP_ID;
+ exportTask.resolve();
+ await completion;
+
+ expect(closeModal).not.toHaveBeenCalled();
+ });
+
+ test('shows an error notification and leaves the modal open when export fails', async () => {
+ const exportTask = deferred();
+ const closeModal = jest.fn();
+ const addNotification = jest.fn();
+ const completion = exportDataFromModal({
+ exportData: () => exportTask.promise,
+ getCurrentModal: () => EXPORT_DATA_ID,
+ closeModal,
+ addNotification
+ });
+
+ exportTask.reject(new Error('Unsupported geometry'));
+ await completion;
+
+ expect(closeModal).not.toHaveBeenCalled();
+ expect(addNotification).toHaveBeenCalledWith({
+ id: 'EXPORT_DATA_ERROR_ID',
+ type: 'error',
+ topic: 'global',
+ message: 'Failed to export data: Unsupported geometry'
+ });
+ });
+});
diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsx
index 77242dd66..e03fb82ff 100644
--- a/src/components/src/modal-container.tsx
+++ b/src/components/src/modal-container.tsx
@@ -125,6 +125,35 @@ export type ModalContainerProps = {
onExportToCloudError?: OnErrorCallBack;
};
+type ExportDataModalActions = {
+ exportData: () => Promise<void>;
+ getCurrentModal: () => UiState['currentModal'];
+ closeModal: () => void;
+ addNotification: ModalContainerProps['uiStateActions']['addNotification'];
+};
+
+export async function exportDataFromModal({
+ exportData: runExport,
+ getCurrentModal,
+ closeModal,
+ addNotification
+}: ExportDataModalActions): Promise<void> {
+ try {
+ await runExport();
+ if (getCurrentModal() === EXPORT_DATA_ID) {
+ closeModal();
+ }
+ } catch (error) {
+ const detail = error instanceof Error ? error.message : String(error);
+ addNotification({
+ id: 'EXPORT_DATA_ERROR_ID',
+ type: 'error',
+ topic: 'global',
+ message: `Failed to export data: ${detail}`
+ });
+ }
+}
+
ModalContainerFactory.deps = [
DeleteDatasetModalFactory,
OverWriteMapModalFactory,
@@ -304,8 +333,12 @@ export default function ModalContainerFactory(
};
_onExportData = () => {
- exportData(this.props, this.props.uiState.exportData);
- this._closeModal();
+ return exportDataFromModal({
+ exportData: () => exportData(this.props, this.props.uiState.exportData),
+ getCurrentModal: () => this.props.uiState.currentModal,
+ closeModal: this._closeModal,
+ addNotification: this.props.uiStateActions.addNotification
+ });
};
_onExportMap = () => {
@@ -412,6 +445,8 @@ export default function ModalContainerFactory(
showDatasetTable={visStateActions.showDatasetTable}
sortTableColumn={visStateActions.sortTableColumn}
pinTableColumn={visStateActions.pinTableColumn}
+ toggleTableColumnHidden={visStateActions.toggleTableColumnHidden}
+ showAllTableColumns={visStateActions.showAllTableColumns}
copyTableColumn={visStateActions.copyTableColumn}
renameTableColumn={visStateActions.renameTableColumn}
deleteTableColumn={visStateActions.deleteTableColumn}
diff --git a/src/components/src/modals/data-table-modal-column-edit.spec.tsx b/src/components/src/modals/data-table-modal-column-edit.spec.tsx
index c6889f40f..0de89921e 100644
--- a/src/components/src/modals/data-table-modal-column-edit.spec.tsx
+++ b/src/components/src/modals/data-table-modal-column-edit.spec.tsx
@@ -7,6 +7,7 @@ import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
import {KeplerTable} from '@kepler.gl/table';
import {renderWithTheme} from 'test/helpers/component-jest-utils';
+import {defaultGetRowCell} from '../common/data-table';
import DataTableModalFactory from './data-table-modal';
jest.mock('../common/data-table/cell-size', () => ({
@@ -30,7 +31,7 @@ function StubDataTable(props: {
const DataTableModal = DataTableModalFactory(StubDataTable as never, (() => null) as never);
-function makeDataset(type: string) {
+function makeDataset(type: string, hiddenColumns: string[] = []) {
const table = new KeplerTable({
info: {id: 'cities', label: 'cities', type},
color: [0, 0, 0]
@@ -38,14 +39,16 @@ function makeDataset(type: string) {
table.updateSchema({
fields: [
{name: 'region', type: ALL_FIELD_TYPES.string},
+ {name: 'secret', type: ALL_FIELD_TYPES.string},
{name: 'pop', type: ALL_FIELD_TYPES.integer}
],
- rows: [['west', 1]]
+ rows: [['west', 'hidden-value', 1]]
});
+ table.hiddenColumns = hiddenColumns;
return table;
}
-function renderModal(type: string) {
+function renderModal(type: string, hiddenColumns: string[] = []) {
const createFilterFromColumn = jest.fn();
const renameTableColumn = jest.fn();
const deleteTableColumn = jest.fn();
@@ -54,7 +57,7 @@ function renderModal(type: string) {
const view = renderWithTheme(
<DataTableModal
dataId="cities"
- datasets={{cities: makeDataset(type)}}
+ datasets={{cities: makeDataset(type, hiddenColumns)}}
showTab={false}
sortTableColumn={jest.fn()}
pinTableColumn={jest.fn()}
@@ -91,4 +94,27 @@ describe('data table column edits', () => {
expect(tableProps.current?.renameTableColumn).toBeUndefined();
expect(tableProps.current?.deleteTableColumn).toBeUndefined();
});
+
+ test('keeps original field indexes when a middle column is hidden', () => {
+ renderModal('local', ['secret']);
+
+ const props = tableProps.current as {
+ columns: string[];
+ colMeta: Record<string, {colIdx: number; type: string}>;
+ dataContainer: {valueAt: (row: number, column: number) => unknown};
+ };
+ expect(props.columns).toEqual(['region', 'pop']);
+ expect(props.colMeta.pop.colIdx).toBe(2);
+ expect(
+ defaultGetRowCell(
+ {
+ ...props,
+ column: 'pop',
+ rowIndex: 0,
+ sortOrder: null
+ } as never,
+ null
+ )
+ ).toBe('1');
+ });
});
diff --git a/src/components/src/modals/data-table-modal.tsx b/src/components/src/modals/data-table-modal.tsx
index 6fcc5092e..9ae8fe29e 100644
--- a/src/components/src/modals/data-table-modal.tsx
+++ b/src/components/src/modals/data-table-modal.tsx
@@ -70,8 +70,19 @@ export const DatasetModalTab: IStyledComponent<
}
`;
+const HiddenColumnsBar = styled.button`
+ margin-right: 12px;
+ border: 0;
+ background: transparent;
+ color: ${props => props.theme.subtextColor};
+ cursor: pointer;
+ font-size: 11px;
+ padding: 0;
+`;
+
const StyledConfigureButton = styled.div`
display: flex;
+ align-items: center;
justify-content: flex-end;
position: absolute;
top: 24px;
@@ -129,6 +140,8 @@ interface DataTableModalProps {
dataId?: string;
sortTableColumn: (id: string, column: string, mode?: string) => void;
pinTableColumn: (id: string, column: string) => void;
+ toggleTableColumnHidden?: (id: string, column: string) => void;
+ showAllTableColumns?: (id: string) => void;
copyTableColumn: (id: string, column: string) => void;
renameTableColumn?: (id: string, column: string, newName: string) => void;
deleteTableColumn?: (id: string, column: string) => void;
@@ -157,16 +170,17 @@ function DataTableModalFactory(
dataId = '',
sortTableColumn,
pinTableColumn,
+ toggleTableColumnHidden,
+ showAllTableColumns,
copyTableColumn: copyTableColumnProp,
renameTableColumn,
deleteTableColumn,
createFilterFromColumn,
datasets,
showDatasetTable,
+ uiStateActions,
showTab = true,
setColumnDisplayFormat: setColumnDisplayFormatProp,
- uiStateActions,
- uiState,
loadColumnStats
}) => {
const [showConfig, setShowConfig] = useState(false);
@@ -179,15 +193,27 @@ function DataTableModalFactory(
[datasets, dataId]
);
- const columns = useMemo(() => fields?.map(f => f.name) || [], [fields]);
+ const hiddenColumns = useMemo(
+ () => datasets?.[dataId]?.hiddenColumns ?? [],
+ [datasets, dataId]
+ );
+ const columns = useMemo(() => {
+ const hidden = new Set(hiddenColumns);
+ return (fields?.map(field => field.name) || []).filter(name => !hidden.has(name));
+ }, [fields, hiddenColumns]);
const colMeta = useMemo(
() =>
fields?.reduce(
- (acc, {name, displayName, type, filterProps, format, displayFormat, isLoadingStats}) => ({
+ (
+ acc,
+ {name, displayName, type, filterProps, format, displayFormat, isLoadingStats},
+ colIdx
+ ) => ({
...acc,
[name]: {
name: displayName || name,
+ colIdx,
type,
...(format ? {format} : {}),
...(displayFormat ? {displayFormat} : {}),
@@ -267,6 +293,17 @@ function DataTableModalFactory(
[pinTableColumn, dataId]
);
+ const handleHideTableColumn = useCallback(
+ (column: string) => {
+ toggleTableColumnHidden?.(dataId, column);
+ },
+ [toggleTableColumnHidden, dataId]
+ );
+
+ const handleShowAllColumns = useCallback(() => {
+ showAllTableColumns?.(dataId);
+ }, [showAllTableColumns, dataId]);
+
const handleSortTableColumn = useCallback(
(column: string, mode?: string) => {
sortTableColumn(dataId, column, mode);
@@ -332,6 +369,11 @@ function DataTableModalFactory(
/>
) : null}
<StyledConfigureButton className="display-config-button">
+ {hiddenColumns.length ? (
+ <HiddenColumnsBar type="button" onClick={handleShowAllColumns}>
+ {`Show ${hiddenColumns.length} hidden`}
+ </HiddenColumnsBar>
+ ) : null}
<Gear onClick={onOpenConfig} />
<Portaled right={240} top={20} isOpened={showConfig} onClose={onCloseConfig}>
<DataTableConfig
@@ -350,11 +392,14 @@ function DataTableModalFactory(
colMeta={colMeta}
cellSizeCache={cellSizeCache}
dataContainer={activeDataset.dataContainer}
- pinnedColumns={activeDataset.pinnedColumns}
+ pinnedColumns={(activeDataset.pinnedColumns || []).filter(
+ column => !hiddenColumns.includes(column)
+ )}
sortOrder={activeDataset.sortOrder}
sortColumn={activeDataset.sortColumn || DEFAULT_SORT_COLUMN}
copyTableColumn={handleCopyTableColumn}
pinTableColumn={handlePinTableColumn}
+ hideTableColumn={toggleTableColumnHidden ? handleHideTableColumn : undefined}
sortTableColumn={handleSortTableColumn}
setColumnDisplayFormat={handleSetColumnDisplayFormat}
renameTableColumn={
diff --git a/src/components/src/modals/export-data-modal.spec.tsx b/src/components/src/modals/export-data-modal.spec.tsx
new file mode 100644
index 000000000..8add9aeff
--- /dev/null
+++ b/src/components/src/modals/export-data-modal.spec.tsx
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import React from 'react';
+import {fireEvent} from '@testing-library/react';
+
+import {EXPORT_DATA_TYPE, EXPORT_DATA_TYPE_OPTIONS} from '@kepler.gl/constants';
+import {Datasets} from '@kepler.gl/table';
+import {renderWithTheme} from 'test/helpers/component-jest-utils';
+
+import ExportDataModalFactory from './export-data-modal';
+
+const ExportDataModal = ExportDataModalFactory();
+
+const datasets = {
+ trips: {
+ label: 'Trips',
+ dataContainer: {numRows: () => 2},
+ filteredIdxCPU: [0]
+ }
+};
+
+describe('ExportDataModal', () => {
+ test('data type is a dropdown that defaults to csv', () => {
+ const onChangeExportDataType = jest.fn();
+ const {container} = renderWithTheme(
+ <ExportDataModal
+ datasets={datasets as unknown as Datasets}
+ selectedDataset="trips"
+ dataType={EXPORT_DATA_TYPE.CSV}
+ filtered={false}
+ applyCPUFilter={jest.fn()}
+ onChangeExportSelectedDataset={jest.fn()}
+ onChangeExportDataType={onChangeExportDataType}
+ onChangeExportFiltered={jest.fn()}
+ />
+ );
+
+ const dataType = container.querySelector('select.data-type-select') as HTMLSelectElement;
+ expect(dataType).not.toBeNull();
+ expect(dataType.value).toBe(EXPORT_DATA_TYPE.CSV);
+ expect(Array.from(dataType.options).map(option => option.text)).toEqual(
+ EXPORT_DATA_TYPE_OPTIONS.map(option => option.label)
+ );
+ expect(dataType.parentElement?.children).toHaveLength(1);
+
+ fireEvent.change(dataType, {target: {value: EXPORT_DATA_TYPE.GEOJSON}});
+ expect(onChangeExportDataType).toHaveBeenCalledWith(EXPORT_DATA_TYPE.GEOJSON);
+ });
+});
diff --git a/src/components/src/modals/export-data-modal.tsx b/src/components/src/modals/export-data-modal.tsx
index a1a7c01bf..b54532906 100644
--- a/src/components/src/modals/export-data-modal.tsx
+++ b/src/components/src/modals/export-data-modal.tsx
@@ -9,12 +9,10 @@ import {DatasetType, EXPORT_DATA_TYPE_OPTIONS} from '@kepler.gl/constants';
import {FormattedMessage} from '@kepler.gl/localization';
import {Datasets} from '@kepler.gl/table';
-import {FileType} from '../common/icons';
import {
StyledExportSection,
StyledFilteredOption,
StyledModalContent,
- StyledType,
CheckMark
} from '../common/styled-components';
import {StyledWarning} from './export-map-modal/components';
@@ -64,7 +62,7 @@ export interface ExportDataModalProps {
onChangeExportDataType: (type: string) => void;
onChangeExportFiltered: (isFiltered: boolean) => void;
intl: IntlShape;
- supportedDataTypes: {
+ supportedDataTypes?: {
id: string;
label: string;
available: boolean;
@@ -95,6 +93,10 @@ const ExportDataModalFactory = () => {
onChangeExportSelectedDataset(value);
};
+ const onSelectDataType: React.ChangeEventHandler<HTMLSelectElement> = ({target: {value}}) => {
+ onChangeExportDataType(value);
+ };
+
const exportAllDatasets = selectedDataset ? !datasets[selectedDataset] : true;
const showTiledDatasetWarning = Object.keys(datasets).some(datasetId => {
return (
@@ -141,16 +143,13 @@ const ExportDataModalFactory = () => {
</div>
</div>
<div className="selection">
- {supportedDataTypes.map(op => (
- <StyledType
- key={op.id}
- selected={dataType === op.id}
- onClick={() => op.available && onChangeExportDataType(op.id)}
- >
- <FileType ext={op.label} height="80px" fontSize="11px" />
- {dataType === op.id && <CheckMark />}
- </StyledType>
- ))}
+ <select className="data-type-select" value={dataType} onChange={onSelectDataType}>
+ {supportedDataTypes.map(op => (
+ <option key={op.id} value={op.id} disabled={!op.available}>
+ {op.label}
+ </option>
+ ))}
+ </select>
</div>
</CompactExportSection>
<CompactExportSection>
diff --git a/src/constants/src/default-settings.ts b/src/constants/src/default-settings.ts
index 9dd350e81..217604142 100644
--- a/src/constants/src/default-settings.ts
+++ b/src/constants/src/default-settings.ts
@@ -539,6 +539,7 @@ export const TABLE_OPTION = keyMirror({
UNPIN: null,
COPY: null,
FORMAT_COLUMN: null,
+ HIDE: null,
RENAME: null,
DELETE: null,
CREATE_FILTER: null
@@ -553,6 +554,7 @@ export type TableOption = {
| 'UNPIN'
| 'COPY'
| 'FORMAT_COLUMN'
+ | 'HIDE'
| 'RENAME'
| 'DELETE'
| 'CREATE_FILTER';
@@ -592,6 +594,12 @@ export const TABLE_OPTION_LIST: TableOption[] = [
condition: props => props.isPinned
},
{value: TABLE_OPTION.COPY, display: 'Copy Column', icon: 'Clipboard'},
+ {
+ value: TABLE_OPTION.HIDE,
+ display: 'Hide Column',
+ icon: 'EyeUnseen',
+ condition: props => props.hideTableColumn
+ },
{
value: TABLE_OPTION.FORMAT_COLUMN,
display: 'Format Column',
@@ -1194,39 +1202,51 @@ export const EXPORT_IMG_RESOLUTION_OPTIONS: ReadonlyArray<ImageResolutionOption>
];
export const EXPORT_DATA_TYPE = keyMirror({
- CSV: null
- // SHAPEFILE: null,
- // JSON: null,
- // GEOJSON: null,
- // TOPOJSON: null
+ CSV: null,
+ GEOJSON: null,
+ GEOPARQUET: null,
+ SHAPEFILE: null,
+ GEOPACKAGE: null,
+ KML: null,
+ KMZ: null
});
export const EXPORT_DATA_TYPE_OPTIONS = [
{
id: EXPORT_DATA_TYPE.CSV,
- label: EXPORT_DATA_TYPE.CSV.toLowerCase(),
+ label: 'csv',
+ available: true
+ },
+ {
+ id: EXPORT_DATA_TYPE.GEOJSON,
+ label: 'geojson',
+ available: true
+ },
+ {
+ id: EXPORT_DATA_TYPE.GEOPARQUET,
+ label: 'parquet',
+ available: true
+ },
+ {
+ id: EXPORT_DATA_TYPE.SHAPEFILE,
+ label: 'shp',
+ available: true
+ },
+ {
+ id: EXPORT_DATA_TYPE.GEOPACKAGE,
+ label: 'gpkg',
+ available: true
+ },
+ {
+ id: EXPORT_DATA_TYPE.KML,
+ label: 'kml',
+ available: true
+ },
+ {
+ id: EXPORT_DATA_TYPE.KMZ,
+ label: 'kmz',
available: true
}
- // {
- // id: EXPORT_DATA_TYPE.SHAPEFILE,
- // label: 'shapefile',
- // available: false
- // },
- // {
- // id: EXPORT_DATA_TYPE.JSON,
- // label: 'json',
- // available: false
- // },
- // {
- // id: EXPORT_DATA_TYPE.GEOJSON,
- // label: 'geojson',
- // available: false
- // },
- // {
- // id: EXPORT_DATA_TYPE.TOPOJSON,
- // label: 'topojson',
- // available: false
- // }
];
// Export map types
diff --git a/src/processors/package.json b/src/processors/package.json
index 3d97730d2..d659796f6 100644
--- a/src/processors/package.json
+++ b/src/processors/package.json
@@ -53,6 +53,8 @@
"@turf/helpers": "^7.2.0",
"apache-arrow": ">=15.0.0",
"d3-dsv": "^2.0.0",
+ "parquet-wasm": "0.6.1",
+ "sql.js": "1.13.0",
"type-analyzer": "0.4.0"
},
"nyc": {
@@ -81,5 +83,8 @@
"require": "./dist/cjs/index.js",
"default": "./dist/cjs/index.js"
}
+ },
+ "devDependencies": {
+ "@types/sql.js": "^1"
}
}
diff --git a/src/processors/src/export/dataset-to-geojson.ts b/src/processors/src/export/dataset-to-geojson.ts
new file mode 100644
index 000000000..55d6fbbe0
--- /dev/null
+++ b/src/processors/src/export/dataset-to-geojson.ts
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {h3IsValid, idToPolygonGeo} from '@kepler.gl/common-utils';
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {parseGeometry} from '@kepler.gl/table';
+import {Field, FieldPair} from '@kepler.gl/types';
+import {DataContainerInterface, parseFieldValue} from '@kepler.gl/utils';
+import {convertWKBToGeometry, convertWKTToGeometry} from '@loaders.gl/gis';
+import type {Feature, FeatureCollection, Geometry, Position} from 'geojson';
+
+type GeometryPlan =
+ | {kind: 'geo'; fieldIndex: number}
+ | {kind: 'latlng'; pair: FieldPair}
+ | {kind: 'h3'; fieldIndex: number}
+ | {kind: 'none'};
+
+function geometryPlan(fields: Field[], fieldPairs: FieldPair[] = []): GeometryPlan {
+ const geoIndex = fields.findIndex(
+ field =>
+ field.type === ALL_FIELD_TYPES.geojson ||
+ field.type === ALL_FIELD_TYPES.geoarrow ||
+ field.type === ALL_FIELD_TYPES.point
+ );
+ if (geoIndex >= 0) {
+ return {kind: 'geo', fieldIndex: geoIndex};
+ }
+ if (fieldPairs[0]) {
+ return {kind: 'latlng', pair: fieldPairs[0]};
+ }
+ const h3Index = fields.findIndex(field => field.type === ALL_FIELD_TYPES.h3);
+ if (h3Index >= 0) {
+ return {kind: 'h3', fieldIndex: h3Index};
+ }
+ return {kind: 'none'};
+}
+
+function geometryFromBinary(raw: ArrayBuffer | ArrayBufferView): Geometry | null {
+ try {
+ const bytes =
+ raw instanceof ArrayBuffer
+ ? raw
+ : raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength);
+ return convertWKBToGeometry(bytes) as Geometry;
+ } catch {
+ return null;
+ }
+}
+
+function geometryFromRaw(raw: unknown, fieldType: string): Geometry | null {
+ if (raw == null) {
+ return null;
+ }
+ if (raw instanceof ArrayBuffer || ArrayBuffer.isView(raw)) {
+ return geometryFromBinary(raw);
+ }
+ const parsed = parseGeometry(raw);
+ if (parsed?.geometry) {
+ return parsed.geometry;
+ }
+ if (fieldType === ALL_FIELD_TYPES.geoarrow && typeof raw === 'string') {
+ const text = raw.trim();
+ if (/^(POINT|LINESTRING|POLYGON|MULTI|GEOMETRYCOLLECTION)\b/i.test(text)) {
+ try {
+ return convertWKTToGeometry(text) as Geometry;
+ } catch {
+ return null;
+ }
+ }
+ }
+ return null;
+}
+
+function attributeValue(value: unknown, field: Field): string | number | boolean | null {
+ if (value == null) {
+ return null;
+ }
+ if (typeof value === 'number') {
+ return Number.isFinite(value) ? value : null;
+ }
+ if (typeof value === 'boolean' || typeof value === 'string') {
+ return value;
+ }
+ if (typeof value === 'bigint') {
+ return value.toString();
+ }
+ const formatted = parseFieldValue(value, field.type, field);
+ return formatted === '' ? null : formatted;
+}
+
+function pointGeometry(
+ data: DataContainerInterface,
+ rowIndex: number,
+ pair: FieldPair
+): Geometry | null {
+ const lat = Number(data.valueAt(rowIndex, pair.pair.lat.fieldIdx));
+ const lng = Number(data.valueAt(rowIndex, pair.pair.lng.fieldIdx));
+ if (!Number.isFinite(lat) || !Number.isFinite(lng)) {
+ return null;
+ }
+ const altitudeIndex = pair.pair.altitude?.fieldIdx;
+ const altitude =
+ altitudeIndex === undefined ? undefined : Number(data.valueAt(rowIndex, altitudeIndex));
+ const coordinates: Position =
+ altitude !== undefined && Number.isFinite(altitude) ? [lng, lat, altitude] : [lng, lat];
+ return {type: 'Point', coordinates};
+}
+
+/**
+ * Turn dataset rows into a FeatureCollection.
+ * Columns listed in `hiddenColumns` are left out of feature properties.
+ * A geometry column is still used to build the feature geometry.
+ */
+export function datasetToFeatureCollection(
+ data: DataContainerInterface,
+ fields: Field[],
+ options: {hiddenColumns?: string[]; fieldPairs?: FieldPair[]} = {}
+): FeatureCollection {
+ const hidden = new Set(options.hiddenColumns ?? []);
+ const plan = geometryPlan(fields, options.fieldPairs);
+ const geometryFieldIndex = plan.kind === 'geo' || plan.kind === 'h3' ? plan.fieldIndex : -1;
+ const features: Feature[] = [];
+
+ for (let rowIndex = 0; rowIndex < data.numRows(); rowIndex++) {
+ const properties: Record<string, string | number | boolean | null> = {};
+ fields.forEach((field, fieldIndex) => {
+ if (hidden.has(field.name) || fieldIndex === geometryFieldIndex) {
+ return;
+ }
+ properties[field.name] = attributeValue(data.valueAt(rowIndex, fieldIndex), field);
+ });
+
+ let geometry: Geometry | null = null;
+ if (plan.kind === 'geo') {
+ geometry = geometryFromRaw(
+ data.valueAt(rowIndex, plan.fieldIndex),
+ fields[plan.fieldIndex].type
+ );
+ } else if (plan.kind === 'latlng') {
+ geometry = pointGeometry(data, rowIndex, plan.pair);
+ } else if (plan.kind === 'h3') {
+ const index = String(data.valueAt(rowIndex, plan.fieldIndex) ?? '');
+ geometry =
+ (h3IsValid(index)
+ ? (idToPolygonGeo({id: index}, {isClosed: true})?.geometry as Geometry | undefined)
+ : null) ?? null;
+ }
+
+ features.push({type: 'Feature', geometry: geometry as Geometry, properties});
+ }
+
+ return {type: 'FeatureCollection', features};
+}
diff --git a/src/processors/src/export/geopackage-writer.ts b/src/processors/src/export/geopackage-writer.ts
new file mode 100644
index 000000000..1f91325a0
--- /dev/null
+++ b/src/processors/src/export/geopackage-writer.ts
@@ -0,0 +1,392 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+// Adapted from GeoLibre (https://github.com/opengeos/GeoLibre), MIT License.
+
+import {convertGeometryToWKB} from '@loaders.gl/gis';
+import type {FeatureCollection, Geometry, Position} from 'geojson';
+import type {Database, SqlJsStatic} from 'sql.js';
+
+import {loadSqlJs} from './sql-js';
+
+/**
+ * Write a GeoJSON FeatureCollection to an OGC GeoPackage (a single SQLite
+ * database) entirely in the browser via sql.js.
+ *
+ * DuckDB-WASM cannot write the SQLite-backed GeoPackage format (its virtual
+ * filesystem lacks the random-access seek/write the driver needs) and Pyodide's
+ * bundled GDAL has no working GeoPackage write driver, so the file is assembled
+ * directly here. Output carries one feature layer in WGS84 (EPSG:4326) with the
+ * required `gpkg_*` metadata tables plus `gpkg_ogr_contents`, so it reopens in
+ * GeoLibre, QGIS, and ArcGIS.
+ */
+
+// SQLite "application_id" for a GeoPackage: ASCII "GPKG" as a big-endian int32.
+const GPKG_APPLICATION_ID = 0x47504b47;
+// SQLite "user_version" for GeoPackage 1.3 (10300), per the spec's versioning.
+const GPKG_USER_VERSION = 10300;
+const SRS_ID = 4326;
+const GEOMETRY_COLUMN = 'geom';
+const FID_COLUMN = 'fid';
+const GPKG_METADATA_TABLES = new Set([
+ 'gpkg_spatial_ref_sys',
+ 'gpkg_contents',
+ 'gpkg_geometry_columns',
+ 'gpkg_ogr_contents'
+]);
+
+const WGS84_DEFINITION =
+ 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,' +
+ 'AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],' +
+ 'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],' +
+ 'UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],' +
+ 'AUTHORITY["EPSG","4326"]]';
+
+type ColumnType = 'INTEGER' | 'REAL' | 'TEXT';
+
+interface ColumnSpec {
+ name: string;
+ type: ColumnType;
+}
+
+interface BoundingBox {
+ minX: number;
+ minY: number;
+ maxX: number;
+ maxY: number;
+}
+
+function emptyBoundingBox(): BoundingBox {
+ return {minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity};
+}
+
+function extendBoundingBox(box: BoundingBox, geometry: Geometry): void {
+ const visit = (position: Position) => {
+ const x = Number(position[0]);
+ const y = Number(position[1]);
+ if (Number.isFinite(x)) {
+ box.minX = Math.min(box.minX, x);
+ box.maxX = Math.max(box.maxX, x);
+ }
+ if (Number.isFinite(y)) {
+ box.minY = Math.min(box.minY, y);
+ box.maxY = Math.max(box.maxY, y);
+ }
+ };
+ const walk = (geom: Geometry) => {
+ switch (geom.type) {
+ case 'Point':
+ visit(geom.coordinates);
+ break;
+ case 'LineString':
+ case 'MultiPoint':
+ geom.coordinates.forEach(visit);
+ break;
+ case 'Polygon':
+ case 'MultiLineString':
+ geom.coordinates.forEach(part => part.forEach(visit));
+ break;
+ case 'MultiPolygon':
+ geom.coordinates.forEach(polygon => polygon.forEach(ring => ring.forEach(visit)));
+ break;
+ case 'GeometryCollection':
+ geom.geometries.forEach(walk);
+ break;
+ default:
+ break;
+ }
+ };
+ walk(geometry);
+}
+
+function quoteIdentifier(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`;
+}
+
+/** Strip characters a SQLite identifier should not carry and avoid collisions
+ * with the reserved fid/geom columns. */
+function sanitizeColumnName(name: string, taken: Set<string>): string {
+ let cleaned = name.replace(/[^\p{L}\p{N}_]+/gu, '_').replace(/^_+/, '');
+ if (!cleaned) cleaned = 'field';
+ if (cleaned.toLowerCase() === FID_COLUMN || cleaned.toLowerCase() === GEOMETRY_COLUMN) {
+ cleaned = `${cleaned}_`;
+ }
+ let candidate = cleaned;
+ let index = 1;
+ while (taken.has(candidate.toLowerCase())) {
+ candidate = `${cleaned}_${index}`;
+ index += 1;
+ }
+ taken.add(candidate.toLowerCase());
+ return candidate;
+}
+
+function sanitizeTableName(name: string): string {
+ const candidate = sanitizeColumnName(name, new Set(GPKG_METADATA_TABLES));
+ return /^sqlite_/i.test(candidate) ? `layer_${candidate}` : candidate;
+}
+
+/** Infer a SQLite column type from a property's values across all features. */
+function inferColumnType(values: unknown[]): ColumnType {
+ let sawNumber = false;
+ let allInteger = true;
+ for (const value of values) {
+ if (value == null) continue;
+ if (typeof value === 'number' && Number.isFinite(value)) {
+ sawNumber = true;
+ if (!Number.isInteger(value)) allInteger = false;
+ } else if (typeof value === 'boolean') {
+ sawNumber = true; // store booleans as 0/1 integers
+ } else {
+ return 'TEXT';
+ }
+ }
+ if (!sawNumber) return 'TEXT';
+ return allInteger ? 'INTEGER' : 'REAL';
+}
+
+/** Coerce a property value to what a SQLite bind expects for `type`. */
+function coerceValue(value: unknown, type: ColumnType): number | string | null {
+ if (value == null) return null;
+ if (type === 'TEXT') {
+ return typeof value === 'object' ? JSON.stringify(value) : String(value);
+ }
+ if (typeof value === 'boolean') return value ? 1 : 0;
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
+ return null;
+}
+
+/**
+ * Build the GeoPackage geometry blob: the "GP" header (version, flags, srs_id)
+ * followed by the standard WKB. Flags `0x01` mean little-endian header fields
+ * and no envelope. Returns null for a null/empty geometry so the row's geom
+ * column is SQL NULL.
+ */
+function geoPackageBlob(geometry: Geometry | null): Uint8Array | null {
+ if (!geometry) return null;
+ const wkb = new Uint8Array(convertGeometryToWKB(geometry));
+ const header = new Uint8Array(8);
+ const view = new DataView(header.buffer);
+ header[0] = 0x47; // 'G'
+ header[1] = 0x50; // 'P'
+ header[2] = 0x00; // version 0
+ header[3] = 0x01; // flags: little-endian, no envelope, standard, non-empty
+ view.setInt32(4, SRS_ID, true);
+ const blob = new Uint8Array(header.length + wkb.length);
+ blob.set(header, 0);
+ blob.set(wkb, header.length);
+ return blob;
+}
+
+function geometryTypeName(types: Set<string>): string {
+ if (types.size === 1) {
+ return [...types][0].toUpperCase();
+ }
+ return 'GEOMETRY';
+}
+
+function createMetadataTables(db: Database): void {
+ db.run(`PRAGMA application_id = ${GPKG_APPLICATION_ID};`);
+ db.run(`PRAGMA user_version = ${GPKG_USER_VERSION};`);
+ db.run(
+ `CREATE TABLE gpkg_spatial_ref_sys (
+ srs_name TEXT NOT NULL,
+ srs_id INTEGER NOT NULL PRIMARY KEY,
+ organization TEXT NOT NULL,
+ organization_coordsys_id INTEGER NOT NULL,
+ definition TEXT NOT NULL,
+ description TEXT
+ );`
+ );
+ const srs = db.prepare(`INSERT INTO gpkg_spatial_ref_sys VALUES (?, ?, ?, ?, ?, ?);`);
+ try {
+ srs.run(['Undefined cartesian SRS', -1, 'NONE', -1, 'undefined', null]);
+ srs.run(['Undefined geographic SRS', 0, 'NONE', 0, 'undefined', null]);
+ srs.run(['WGS 84 geodetic', SRS_ID, 'EPSG', SRS_ID, WGS84_DEFINITION, null]);
+ } finally {
+ srs.free();
+ }
+
+ db.run(
+ `CREATE TABLE gpkg_contents (
+ table_name TEXT NOT NULL PRIMARY KEY,
+ data_type TEXT NOT NULL,
+ identifier TEXT UNIQUE,
+ description TEXT DEFAULT '',
+ last_change DATETIME NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+ min_x DOUBLE,
+ min_y DOUBLE,
+ max_x DOUBLE,
+ max_y DOUBLE,
+ srs_id INTEGER,
+ CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id)
+ REFERENCES gpkg_spatial_ref_sys(srs_id)
+ );`
+ );
+ db.run(
+ `CREATE TABLE gpkg_geometry_columns (
+ table_name TEXT NOT NULL,
+ column_name TEXT NOT NULL,
+ geometry_type_name TEXT NOT NULL,
+ srs_id INTEGER NOT NULL,
+ z TINYINT NOT NULL,
+ m TINYINT NOT NULL,
+ CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name),
+ CONSTRAINT uk_gc_table_name UNIQUE (table_name),
+ CONSTRAINT fk_gc_tn FOREIGN KEY (table_name)
+ REFERENCES gpkg_contents(table_name),
+ CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id)
+ REFERENCES gpkg_spatial_ref_sys(srs_id)
+ );`
+ );
+ db.run(
+ `CREATE TABLE gpkg_ogr_contents (
+ table_name TEXT NOT NULL PRIMARY KEY,
+ feature_count INTEGER DEFAULT NULL
+ );`
+ );
+}
+
+/**
+ * Synchronous core of {@link writeGeoPackage}, separated so it can be unit-tested
+ * with an already-initialised sql.js factory (the async wrapper resolves the
+ * factory through a Vite `?url` import that only works in the browser).
+ *
+ * Args:
+ * SQL: An initialised sql.js factory.
+ * geojson: Features to write (assumed WGS84).
+ * layerName: Name of the single feature table / layer in the GeoPackage.
+ *
+ * Returns:
+ * The GeoPackage file as bytes.
+ */
+export function writeGeoPackageSync(
+ SQL: SqlJsStatic,
+ geojson: FeatureCollection,
+ layerName: string
+): Uint8Array {
+ const features = geojson.features ?? [];
+ if (features.length === 0) {
+ throw new Error('The layer has no features to export.');
+ }
+
+ const tableName = sanitizeTableName(layerName);
+
+ // Collect the union of property keys (features may carry different keys).
+ const propertyKeys: string[] = [];
+ const seenKeys = new Set<string>();
+ for (const feature of features) {
+ for (const key of Object.keys(feature.properties ?? {})) {
+ if (!seenKeys.has(key)) {
+ seenKeys.add(key);
+ propertyKeys.push(key);
+ }
+ }
+ }
+
+ const takenColumns = new Set<string>([FID_COLUMN, GEOMETRY_COLUMN]);
+ const columns: (ColumnSpec & {key: string})[] = propertyKeys.map(key => ({
+ key,
+ name: sanitizeColumnName(key, takenColumns),
+ type: inferColumnType(features.map(f => f.properties?.[key]))
+ }));
+
+ const box: BoundingBox = emptyBoundingBox();
+ const geometryTypes = new Set<string>();
+ for (const feature of features) {
+ if (feature.geometry) {
+ geometryTypes.add(feature.geometry.type);
+ extendBoundingBox(box, feature.geometry);
+ }
+ }
+
+ const db = new SQL.Database();
+ try {
+ createMetadataTables(db);
+
+ const columnDefs = columns
+ .map(column => `${quoteIdentifier(column.name)} ${column.type}`)
+ .join(', ');
+ db.run(
+ `CREATE TABLE ${quoteIdentifier(tableName)} (
+ ${quoteIdentifier(FID_COLUMN)} INTEGER PRIMARY KEY AUTOINCREMENT,
+ ${quoteIdentifier(GEOMETRY_COLUMN)} BLOB${columnDefs ? `, ${columnDefs}` : ''}
+ );`
+ );
+
+ const insertColumns = [GEOMETRY_COLUMN, ...columns.map(column => column.name)];
+ const placeholders = insertColumns.map(() => '?').join(', ');
+ const insert = db.prepare(
+ `INSERT INTO ${quoteIdentifier(tableName)} (${insertColumns
+ .map(quoteIdentifier)
+ .join(', ')}) VALUES (${placeholders});`
+ );
+ try {
+ db.run('BEGIN;');
+ for (const feature of features) {
+ const row: (number | string | Uint8Array | null)[] = [geoPackageBlob(feature.geometry)];
+ for (const column of columns) {
+ row.push(coerceValue(feature.properties?.[column.key], column.type));
+ }
+ insert.run(row);
+ }
+ db.run('COMMIT;');
+ } finally {
+ insert.free();
+ }
+
+ const finiteBox = Number.isFinite(box.minX) && Number.isFinite(box.minY);
+ const contents = db.prepare(
+ `INSERT INTO gpkg_contents
+ (table_name, data_type, identifier, description, min_x, min_y, max_x, max_y, srs_id)
+ VALUES (?, 'features', ?, '', ?, ?, ?, ?, ?);`
+ );
+ try {
+ contents.run([
+ tableName,
+ tableName,
+ finiteBox ? box.minX : null,
+ finiteBox ? box.minY : null,
+ finiteBox ? box.maxX : null,
+ finiteBox ? box.maxY : null,
+ SRS_ID
+ ]);
+ } finally {
+ contents.free();
+ }
+
+ db.run(`INSERT INTO gpkg_geometry_columns VALUES (?, ?, ?, ?, 0, 0);`, [
+ tableName,
+ GEOMETRY_COLUMN,
+ geometryTypeName(geometryTypes),
+ SRS_ID
+ ]);
+ db.run(`INSERT INTO gpkg_ogr_contents (table_name, feature_count) VALUES (?, ?);`, [
+ tableName,
+ features.length
+ ]);
+
+ // sql.js always exports an ArrayBuffer-backed Uint8Array.
+ return db.export();
+ } finally {
+ db.close();
+ }
+}
+
+/**
+ * Write a GeoJSON FeatureCollection to a GeoPackage (loads sql.js, then defers
+ * to {@link writeGeoPackageSync}).
+ *
+ * Args:
+ * geojson: Features to write (assumed WGS84).
+ * layerName: Name of the single feature table / layer in the GeoPackage.
+ *
+ * Returns:
+ * The GeoPackage file as bytes.
+ */
+export async function writeGeoPackage(
+ geojson: FeatureCollection,
+ layerName: string
+): Promise<Uint8Array> {
+ const SQL = await loadSqlJs();
+ return writeGeoPackageSync(SQL, geojson, layerName);
+}
diff --git a/src/processors/src/export/geoparquet-writer.ts b/src/processors/src/export/geoparquet-writer.ts
new file mode 100644
index 000000000..9a2dee705
--- /dev/null
+++ b/src/processors/src/export/geoparquet-writer.ts
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {convertGeometryToWKB} from '@loaders.gl/gis';
+import * as arrow from 'apache-arrow';
+import type {FeatureCollection, Geometry} from 'geojson';
+
+const GEOPARQUET_GEOMETRY_TYPES = new Set([
+ 'Point',
+ 'LineString',
+ 'Polygon',
+ 'MultiPoint',
+ 'MultiLineString',
+ 'MultiPolygon',
+ 'GeometryCollection'
+]);
+
+type ParquetWasm = typeof import('parquet-wasm');
+
+/** Keep in sync with the parquet-wasm dependency in package.json. */
+const PARQUET_WASM_URL = 'https://unpkg.com/[email protected]/esm/parquet_wasm_bg.wasm';
+
+let parquetWasmPromise: Promise<ParquetWasm> | null = null;
+
+async function importParquetWasm(): Promise<ParquetWasm> {
+ // Jest uses the CommonJS build, which reads the wasm from disk. A literal
+ // import of that build makes esbuild pull in Node's fs/path. The browser
+ // bundle only keeps the ESM build below, because NODE_ENV is not "test".
+ if (process.env.NODE_ENV === 'test') {
+ const specifier = 'parquet-wasm/node';
+ const imported = (await import(specifier)) as ParquetWasm;
+ return (imported.Table ? imported : imported.default) as ParquetWasm;
+ }
+ const imported = await import('parquet-wasm');
+ if (typeof imported.default === 'function') {
+ await imported.default(PARQUET_WASM_URL);
+ }
+ return (imported.Table ? imported : imported.default) as ParquetWasm;
+}
+
+function loadParquetWasm(): Promise<ParquetWasm> {
+ if (parquetWasmPromise) {
+ return parquetWasmPromise;
+ }
+ const pending = importParquetWasm().catch(error => {
+ parquetWasmPromise = null;
+ throw error;
+ });
+ parquetWasmPromise = pending;
+ return pending;
+}
+
+function wkbBytes(geometry: Geometry | null): Uint8Array | null {
+ if (!geometry) {
+ return null;
+ }
+ return new Uint8Array(convertGeometryToWKB(geometry));
+}
+
+function propertyVector(values: (string | number | boolean | null)[]): arrow.Vector {
+ // tableFromArrays stores JS strings as Dictionary<Int32, Utf8>. Parquet import
+ // rejects that type with "arrow type not supported: Dictionary".
+ let kind: 'bool' | 'float' | 'string' = 'string';
+ let seen = false;
+ for (const value of values) {
+ if (value == null) {
+ continue;
+ }
+ let next: 'bool' | 'float' | 'string' = 'string';
+ if (typeof value === 'boolean') {
+ next = 'bool';
+ } else if (typeof value === 'number') {
+ next = 'float';
+ }
+ if (!seen) {
+ kind = next;
+ seen = true;
+ } else if (kind !== next) {
+ kind = 'string';
+ break;
+ }
+ }
+ if (kind === 'bool') {
+ return arrow.vectorFromArray(values as (boolean | null)[], new arrow.Bool());
+ }
+ if (kind === 'float') {
+ return arrow.vectorFromArray(values as (number | null)[], new arrow.Float64());
+ }
+ return arrow.vectorFromArray(
+ values.map(value => (value == null ? null : String(value))),
+ new arrow.Utf8()
+ );
+}
+
+function parquetValue(value: unknown): string | number | boolean | null {
+ if (value == null) {
+ return null;
+ }
+ if (typeof value === 'number') {
+ return Number.isFinite(value) ? value : null;
+ }
+ if (typeof value === 'boolean' || typeof value === 'string') {
+ return value;
+ }
+ return JSON.stringify(value);
+}
+
+function propertyName(name: string, taken: Set<string>): string {
+ let candidate = name === 'geometry' ? 'geometry_attr' : name;
+ if (!candidate) {
+ candidate = 'field';
+ }
+ let unique = candidate;
+ let index = 1;
+ while (taken.has(unique)) {
+ unique = `${candidate}_${index}`;
+ index += 1;
+ }
+ taken.add(unique);
+ return unique;
+}
+
+/**
+ * Write a GeoJSON FeatureCollection as GeoParquet 1.1 (WKB geometry, WGS84).
+ */
+export async function writeGeoParquet(geojson: FeatureCollection): Promise<Uint8Array> {
+ const features = geojson.features ?? [];
+ const taken = new Set<string>(['geometry']);
+ const propertyKeys: {source: string; name: string}[] = [];
+ const seen = new Set<string>();
+ for (const feature of features) {
+ for (const key of Object.keys(feature.properties ?? {})) {
+ if (!seen.has(key)) {
+ seen.add(key);
+ propertyKeys.push({source: key, name: propertyName(key, taken)});
+ }
+ }
+ }
+
+ const columns: Record<string, arrow.Vector> = {
+ geometry: arrow.vectorFromArray(
+ features.map(feature => wkbBytes(feature.geometry)),
+ new arrow.Binary()
+ )
+ };
+ for (const key of propertyKeys) {
+ columns[key.name] = propertyVector(
+ features.map(feature => parquetValue(feature.properties?.[key.source]))
+ );
+ }
+
+ const table = new arrow.Table(columns);
+ const ipc = arrow.tableToIPC(table, 'stream');
+ const parquetWasm = await loadParquetWasm();
+ const wasmTable = parquetWasm.Table.fromIPCStream(ipc);
+ const geometryTypes: string[] = [];
+ for (const feature of features) {
+ const type = feature.geometry?.type;
+ if (type && GEOPARQUET_GEOMETRY_TYPES.has(type) && !geometryTypes.includes(type)) {
+ geometryTypes.push(type);
+ }
+ }
+ const geoMetadata = JSON.stringify({
+ version: '1.1.0',
+ primary_column: 'geometry',
+ columns: {
+ geometry: {
+ encoding: 'WKB',
+ ...(geometryTypes.length ? {geometry_types: geometryTypes} : {})
+ }
+ }
+ });
+ const properties = new parquetWasm.WriterPropertiesBuilder()
+ .setKeyValueMetadata(new Map([['geo', geoMetadata]]))
+ .build();
+ // writeParquet takes ownership of both objects.
+ const bytes = parquetWasm.writeParquet(wasmTable, properties);
+ return new Uint8Array(bytes);
+}
diff --git a/src/processors/src/export/kml-writer.ts b/src/processors/src/export/kml-writer.ts
new file mode 100644
index 000000000..99c7f73d5
--- /dev/null
+++ b/src/processors/src/export/kml-writer.ts
@@ -0,0 +1,318 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+// Adapted from GeoLibre (https://github.com/opengeos/GeoLibre), MIT License.
+
+import type {Feature, FeatureCollection, GeoJsonProperties, Geometry, Position} from 'geojson';
+export class KmlCoordinateError extends Error {
+ constructor(featureIndex: number, featureId?: string | number) {
+ super(`Feature ${featureId ?? featureIndex} has a coordinate that KML cannot represent.`);
+ this.name = 'KmlCoordinateError';
+ }
+}
+
+const KML_NAMESPACE = 'http://www.opengis.net/kml/2.2';
+
+class InvalidCoordinateError extends Error {}
+
+function xmlSafeText(value: unknown): string {
+ return stripDisallowedControls(String(value))
+ .replace(/&/g, '&amp;')
+ .replace(/</g, '&lt;')
+ .replace(/>/g, '&gt;')
+ .replace(/"/g, '&quot;')
+ .replace(/'/g, '&apos;');
+}
+
+/** XML 1.0 disallows these controls. Tab, newline, and carriage return stay. */
+function stripDisallowedControls(value: string): string {
+ let result = '';
+ for (const char of value) {
+ const code = char.charCodeAt(0);
+ const disallowed =
+ code <= 0x08 || code === 0x0b || code === 0x0c || (code >= 0x0e && code <= 0x1f);
+ result += disallowed ? '\uFFFD' : char;
+ }
+ return result;
+}
+
+function indentXml(xml: string, spaces: number): string {
+ const indentation = ' '.repeat(spaces);
+ return xml
+ .split('\n')
+ .map(line => `${indentation}${line}`)
+ .join('\n');
+}
+
+function decimalText(value: number): string {
+ const text = String(value);
+ const match = /^(-?)(\d+)(?:\.(\d+))?e([+-]?\d+)$/i.exec(text);
+ if (!match) return text;
+
+ const [, sign, integer, fraction = '', rawExponent] = match;
+ const digits = `${integer}${fraction}`;
+ const decimalIndex = integer.length + Number(rawExponent);
+ if (decimalIndex <= 0) {
+ return `${sign}0.${'0'.repeat(-decimalIndex)}${digits}`;
+ }
+ if (decimalIndex >= digits.length) {
+ return `${sign}${digits}${'0'.repeat(decimalIndex - digits.length)}`;
+ }
+ return `${sign}${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;
+}
+
+function positionText(position: Position): string {
+ if (position.length < 2 || position.slice(0, 3).some(value => !Number.isFinite(value))) {
+ throw new InvalidCoordinateError();
+ }
+ return position
+ .slice(0, 3)
+ .map(value => decimalText(value))
+ .join(',');
+}
+
+function coordinatesText(positions: Position[]): string {
+ return positions.map(positionText).join(' ');
+}
+
+function hasAltitude(position: Position): boolean {
+ // KML applies altitude mode to the entire geometry. For mixed-dimensional
+ // GeoJSON, preserve supplied altitudes and let KML interpret 2D vertices as
+ // having its implicit zero altitude.
+ return position.length >= 3;
+}
+
+function closedRing(ring: Position[]): Position[] {
+ if (ring.length === 0) return ring;
+ const first = ring[0];
+ const last = ring[ring.length - 1];
+ if (
+ first.length === last.length &&
+ first.every((coordinate, index) => coordinate === last[index])
+ ) {
+ return ring;
+ }
+ return [...ring, first];
+}
+
+function polygonKml(coordinates: Position[][]): string {
+ const [outer, ...inner] = coordinates;
+ const boundaries: string[] = [];
+ if (outer) {
+ boundaries.push(
+ '<outerBoundaryIs>',
+ ' <LinearRing>',
+ ` <coordinates>${coordinatesText(closedRing(outer))}</coordinates>`,
+ ' </LinearRing>',
+ '</outerBoundaryIs>'
+ );
+ }
+ for (const ring of inner) {
+ boundaries.push(
+ '<innerBoundaryIs>',
+ ' <LinearRing>',
+ ` <coordinates>${coordinatesText(closedRing(ring))}</coordinates>`,
+ ' </LinearRing>',
+ '</innerBoundaryIs>'
+ );
+ }
+ if (boundaries.length === 0) return '<Polygon />';
+ const contents = coordinates.some(ring => ring.some(hasAltitude))
+ ? ['<altitudeMode>absolute</altitudeMode>', ...boundaries]
+ : boundaries;
+ return `<Polygon>\n${indentXml(contents.join('\n'), 2)}\n</Polygon>`;
+}
+
+function multiGeometryKml(geometries: Geometry[]): string {
+ if (geometries.length === 0) return '<MultiGeometry />';
+ return `<MultiGeometry>\n${geometries
+ .map(geometry => indentXml(geometryKml(geometry), 2))
+ .join('\n')}\n</MultiGeometry>`;
+}
+
+function geometryKml(geometry: Geometry): string {
+ switch (geometry.type) {
+ case 'Point':
+ return `<Point>${
+ hasAltitude(geometry.coordinates) ? '<altitudeMode>absolute</altitudeMode>' : ''
+ }<coordinates>${positionText(geometry.coordinates)}</coordinates></Point>`;
+ case 'MultiPoint':
+ return multiGeometryKml(
+ geometry.coordinates.map(coordinates => ({type: 'Point', coordinates}))
+ );
+ case 'LineString':
+ return `<LineString>${
+ geometry.coordinates.some(hasAltitude) ? '<altitudeMode>absolute</altitudeMode>' : ''
+ }<coordinates>${coordinatesText(geometry.coordinates)}</coordinates></LineString>`;
+ case 'MultiLineString':
+ return multiGeometryKml(
+ geometry.coordinates.map(coordinates => ({type: 'LineString', coordinates}))
+ );
+ case 'Polygon':
+ return polygonKml(geometry.coordinates);
+ case 'MultiPolygon':
+ return multiGeometryKml(
+ geometry.coordinates.map(coordinates => ({type: 'Polygon', coordinates}))
+ );
+ case 'GeometryCollection':
+ return multiGeometryKml(geometry.geometries);
+ }
+}
+
+function propertyText(value: unknown): string {
+ if (value == null) return '';
+ if (typeof value !== 'object') return String(value);
+ return JSON.stringify(value);
+}
+
+function propertyElement(properties: GeoJsonProperties, key: string): string | null {
+ const value = properties?.[key];
+ return value == null || typeof value === 'object'
+ ? null
+ : `<${key}>${xmlSafeText(value)}</${key}>`;
+}
+
+function extendedDataKml(feature: Feature): string | null {
+ const properties = feature.properties ?? {};
+ const entries = Object.entries(properties).filter(
+ ([key, value]) =>
+ !((key === 'name' || key === 'description') && value != null && typeof value !== 'object')
+ );
+ if (feature.id != null) {
+ let idKey = 'feature_id';
+ if (Object.prototype.hasOwnProperty.call(properties, idKey)) {
+ idKey = 'geojson_feature_id';
+ let suffix = 2;
+ while (Object.prototype.hasOwnProperty.call(properties, idKey)) {
+ idKey = `geojson_feature_id_${suffix}`;
+ suffix += 1;
+ }
+ }
+ entries.unshift([idKey, feature.id]);
+ }
+ if (entries.length === 0) return null;
+ const data = entries.map(
+ ([key, value]) =>
+ `<Data name="${xmlSafeText(key)}"><value>${xmlSafeText(propertyText(value))}</value></Data>`
+ );
+ return `<ExtendedData>\n${data.map(entry => indentXml(entry, 2)).join('\n')}\n</ExtendedData>`;
+}
+
+function opacityByte(value: unknown, defaultOpacity = 1): string {
+ const opacity = typeof value === 'number' ? value : Number(value ?? defaultOpacity);
+ const normalized = Number.isFinite(opacity) ? Math.min(1, Math.max(0, opacity)) : defaultOpacity;
+ return Math.round(normalized * 255)
+ .toString(16)
+ .padStart(2, '0');
+}
+
+function kmlColor(color: unknown, opacity: unknown, defaultOpacity = 1): string | null {
+ if (typeof color !== 'string') return null;
+ const match = /^#?([0-9a-f]{3}|[0-9a-f]{6})$/i.exec(color.trim());
+ if (!match) return null;
+ const hex =
+ match[1].length === 3
+ ? match[1]
+ .split('')
+ .map(digit => `${digit}${digit}`)
+ .join('')
+ : match[1];
+ const red = hex.slice(0, 2);
+ const green = hex.slice(2, 4);
+ const blue = hex.slice(4, 6);
+ return `${opacityByte(opacity, defaultOpacity)}${blue}${green}${red}`.toLowerCase();
+}
+
+function styleKml(properties: GeoJsonProperties): string | null {
+ if (!properties) return null;
+ const sections: string[] = [];
+
+ const markerColor = kmlColor(properties['marker-color'], properties['marker-opacity']);
+ if (markerColor) {
+ sections.push(`<IconStyle><color>${markerColor}</color></IconStyle>`);
+ }
+
+ const strokeColor = kmlColor(properties.stroke, properties['stroke-opacity']);
+ const strokeWidth = Number(properties['stroke-width']);
+ if (strokeColor || Number.isFinite(strokeWidth)) {
+ const values = [
+ strokeColor ? `<color>${strokeColor}</color>` : null,
+ Number.isFinite(strokeWidth) ? `<width>${strokeWidth}</width>` : null
+ ].filter((value): value is string => value !== null);
+ sections.push(`<LineStyle>${values.join('')}</LineStyle>`);
+ }
+
+ const fillColor = kmlColor(properties.fill, properties['fill-opacity'], 0.6);
+ if (fillColor) {
+ sections.push(`<PolyStyle><color>${fillColor}</color></PolyStyle>`);
+ }
+
+ if (sections.length === 0) return null;
+ return `<Style>${sections.join('')}</Style>`;
+}
+
+function featureKml(feature: Feature, style: string | null): string {
+ const properties = feature.properties ?? {};
+ const contents = [
+ propertyElement(properties, 'name'),
+ propertyElement(properties, 'description'),
+ style,
+ extendedDataKml(feature),
+ feature.geometry ? geometryKml(feature.geometry) : null
+ ].filter((value): value is string => value !== null);
+
+ if (contents.length === 0) return '<Placemark />';
+ return `<Placemark>\n${contents.map(value => indentXml(value, 2)).join('\n')}\n</Placemark>`;
+}
+
+/**
+ * Convert a GeoJSON FeatureCollection to a KML 2.2 document.
+ *
+ * Feature properties are retained in ExtendedData, except primitive `name` and
+ * `description` values that are represented by their native KML elements.
+ * Simplestyle properties are also promoted to native KML styles for display in
+ * Google Earth and other KML clients.
+ */
+export function writeKml(geojson: FeatureCollection, documentName: string): string {
+ const featureStyles = geojson.features.map(feature => styleKml(feature.properties));
+ const styleCounts = new Map<string, number>();
+ for (const style of featureStyles) {
+ if (style) styleCounts.set(style, (styleCounts.get(style) ?? 0) + 1);
+ }
+
+ const sharedStyles = new Map<string, string>();
+ for (const [style, count] of styleCounts) {
+ if (count > 1) sharedStyles.set(style, `style-${sharedStyles.size + 1}`);
+ }
+ const styleDefinitions = Array.from(sharedStyles, ([style, id]) =>
+ indentXml(style.replace('<Style>', `<Style id="${id}">`), 4)
+ );
+
+ const placemarks = geojson.features.map((feature, index) => {
+ try {
+ const style = featureStyles[index];
+ const sharedStyleId = style ? sharedStyles.get(style) : undefined;
+ return indentXml(
+ featureKml(feature, sharedStyleId ? `<styleUrl>#${sharedStyleId}</styleUrl>` : style),
+ 4
+ );
+ } catch (error) {
+ if (error instanceof InvalidCoordinateError) {
+ throw new KmlCoordinateError(index, feature.id);
+ }
+ throw error;
+ }
+ });
+ const documentContents = [
+ ` <name>${xmlSafeText(documentName)}</name>`,
+ ...styleDefinitions,
+ ...placemarks
+ ].join('\n');
+ return [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ `<kml xmlns="${KML_NAMESPACE}">`,
+ ' <Document>',
+ documentContents,
+ ' </Document>',
+ '</kml>'
+ ].join('\n');
+}
diff --git a/src/processors/src/export/shapefile-writer.ts b/src/processors/src/export/shapefile-writer.ts
new file mode 100644
index 000000000..5990c350a
--- /dev/null
+++ b/src/processors/src/export/shapefile-writer.ts
@@ -0,0 +1,510 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+// Adapted from GeoLibre (https://github.com/opengeos/GeoLibre), MIT License.
+
+import type {Feature, FeatureCollection, Geometry, Position} from 'geojson';
+
+/**
+ * Write a GeoJSON FeatureCollection to an ESRI Shapefile dataset (the .shp,
+ * .shx, .dbf, .prj, and .cpg parts) entirely in the browser.
+ *
+ * DuckDB-WASM and Pyodide's bundled GDAL cannot write GDAL vector formats in the
+ * browser, so the binary parts are assembled directly. The dataset is WGS84
+ * (EPSG:4326). All non-null geometries must share one shape family
+ * (point/line/polygon). Null geometries are written as the Shapefile "Null"
+ * shape so the record count still lines up with the .dbf.
+ */
+
+export interface ShapefileParts {
+ shp: Uint8Array;
+ shx: Uint8Array;
+ dbf: Uint8Array;
+ prj: Uint8Array;
+ cpg: Uint8Array;
+}
+
+const SHAPE_NULL = 0;
+const SHAPE_POINT = 1;
+const SHAPE_POLYLINE = 3;
+const SHAPE_POLYGON = 5;
+const SHAPE_MULTIPOINT = 8;
+
+const WGS84_PRJ =
+ 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",' +
+ '6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],' +
+ 'UNIT["Degree",0.0174532925199433]]';
+
+type Family = 'point' | 'line' | 'polygon';
+
+function familyOf(geometry: Geometry): Family | null {
+ switch (geometry.type) {
+ case 'Point':
+ case 'MultiPoint':
+ return 'point';
+ case 'LineString':
+ case 'MultiLineString':
+ return 'line';
+ case 'Polygon':
+ case 'MultiPolygon':
+ return 'polygon';
+ default:
+ return null;
+ }
+}
+
+/** Pick the single Shapefile shape type for the whole file. */
+function fileShapeType(features: Feature[]): {type: number; family: Family} {
+ let family: Family | null = null;
+ let hasMultiPoint = false;
+ for (const feature of features) {
+ if (!feature.geometry) continue;
+ const geomFamily = familyOf(feature.geometry);
+ if (!geomFamily) {
+ throw new Error(`Shapefile export does not support ${feature.geometry.type} geometries.`);
+ }
+ if (family !== null && family !== geomFamily) {
+ throw new Error('Shapefile export requires all geometries to have the same shape family.');
+ }
+ if (family === null) family = geomFamily;
+ if (feature.geometry.type === 'MultiPoint') hasMultiPoint = true;
+ }
+ if (family === null) {
+ throw new Error('The layer has no supported geometries to export.');
+ }
+ if (family === 'point') {
+ return {type: hasMultiPoint ? SHAPE_MULTIPOINT : SHAPE_POINT, family};
+ }
+ return {type: family === 'line' ? SHAPE_POLYLINE : SHAPE_POLYGON, family};
+}
+
+/** Signed area of a ring; positive is counter-clockwise (screen coords, y up). */
+function ringSignedArea(ring: Position[]): number {
+ let area = 0;
+ for (let i = 0; i < ring.length - 1; i += 1) {
+ const [x1, y1] = ring[i];
+ const [x2, y2] = ring[i + 1];
+ area += Number(x1) * Number(y2) - Number(x2) * Number(y1);
+ }
+ return area / 2;
+}
+
+/** Return a ring oriented clockwise (`wantClockwise`) or counter-clockwise.
+ * Shapefile polygons require clockwise outer rings and counter-clockwise holes. */
+function orientRing(ring: Position[], wantClockwise: boolean): Position[] {
+ const isClockwise = ringSignedArea(ring) < 0;
+ return isClockwise === wantClockwise ? ring : [...ring].reverse();
+}
+
+/** Flatten a geometry into Shapefile "parts" (rings/lines) of positions, with
+ * polygon rings reoriented for the Shapefile convention. */
+function geometryParts(geometry: Geometry, family: Family): Position[][] {
+ if (family === 'line') {
+ if (geometry.type === 'LineString') return [geometry.coordinates];
+ if (geometry.type === 'MultiLineString') return geometry.coordinates;
+ return [];
+ }
+ // polygon
+ const polygons =
+ geometry.type === 'Polygon'
+ ? [geometry.coordinates]
+ : geometry.type === 'MultiPolygon'
+ ? geometry.coordinates
+ : [];
+ const parts: Position[][] = [];
+ for (const rings of polygons) {
+ rings.forEach((ring, index) => {
+ // First ring of each polygon is the outer ring (clockwise); the rest are
+ // holes (counter-clockwise).
+ parts.push(orientRing(ring, index === 0));
+ });
+ }
+ return parts;
+}
+
+function pointsOf(geometry: Geometry): Position[] {
+ if (geometry.type === 'Point') return [geometry.coordinates];
+ if (geometry.type === 'MultiPoint') return geometry.coordinates;
+ return [];
+}
+
+interface Box {
+ minX: number;
+ minY: number;
+ maxX: number;
+ maxY: number;
+}
+
+function emptyBox(): Box {
+ return {minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity};
+}
+
+function extend(box: Box, x: number, y: number): void {
+ if (Number.isFinite(x)) {
+ box.minX = Math.min(box.minX, x);
+ box.maxX = Math.max(box.maxX, x);
+ }
+ if (Number.isFinite(y)) {
+ box.minY = Math.min(box.minY, y);
+ box.maxY = Math.max(box.maxY, y);
+ }
+}
+
+function safeBox(box: Box): Box {
+ return Number.isFinite(box.minX) ? box : {minX: 0, minY: 0, maxX: 0, maxY: 0};
+}
+
+/** Encode one geometry's Shapefile record content (without the 8-byte record
+ * header). Returns the content bytes and accumulates into the file bbox. */
+function encodeShape(
+ geometry: Geometry | null,
+ shapeType: number,
+ family: Family,
+ fileBox: Box
+): Uint8Array {
+ if (!geometry || familyOf(geometry) !== family) {
+ // Null shape: just the int32 little-endian shape type 0.
+ const nullShape = new Uint8Array(4);
+ new DataView(nullShape.buffer).setInt32(0, SHAPE_NULL, true);
+ return nullShape;
+ }
+
+ if (shapeType === SHAPE_POINT) {
+ const [x, y] = pointsOf(geometry)[0] ?? [0, 0];
+ extend(fileBox, Number(x), Number(y));
+ const content = new Uint8Array(20);
+ const view = new DataView(content.buffer);
+ view.setInt32(0, SHAPE_POINT, true);
+ view.setFloat64(4, Number(x), true);
+ view.setFloat64(12, Number(y), true);
+ return content;
+ }
+
+ if (shapeType === SHAPE_MULTIPOINT) {
+ const points = pointsOf(geometry);
+ const box = emptyBox();
+ for (const [x, y] of points) extend(box, Number(x), Number(y));
+ const drawn = safeBox(box);
+ extend(fileBox, drawn.minX, drawn.minY);
+ extend(fileBox, drawn.maxX, drawn.maxY);
+ const content = new Uint8Array(4 + 32 + 4 + points.length * 16);
+ const view = new DataView(content.buffer);
+ view.setInt32(0, SHAPE_MULTIPOINT, true);
+ view.setFloat64(4, drawn.minX, true);
+ view.setFloat64(12, drawn.minY, true);
+ view.setFloat64(20, drawn.maxX, true);
+ view.setFloat64(28, drawn.maxY, true);
+ view.setInt32(36, points.length, true);
+ let offset = 40;
+ for (const [x, y] of points) {
+ view.setFloat64(offset, Number(x), true);
+ view.setFloat64(offset + 8, Number(y), true);
+ offset += 16;
+ }
+ return content;
+ }
+
+ // PolyLine (3) or Polygon (5): identical record layout.
+ const parts = geometryParts(geometry, family);
+ const numParts = parts.length;
+ const flatPoints: Position[] = parts.flat();
+ const box = emptyBox();
+ for (const [x, y] of flatPoints) extend(box, Number(x), Number(y));
+ const drawn = safeBox(box);
+ extend(fileBox, drawn.minX, drawn.minY);
+ extend(fileBox, drawn.maxX, drawn.maxY);
+
+ const content = new Uint8Array(4 + 32 + 4 + 4 + numParts * 4 + flatPoints.length * 16);
+ const view = new DataView(content.buffer);
+ view.setInt32(0, shapeType, true);
+ view.setFloat64(4, drawn.minX, true);
+ view.setFloat64(12, drawn.minY, true);
+ view.setFloat64(20, drawn.maxX, true);
+ view.setFloat64(28, drawn.maxY, true);
+ view.setInt32(36, numParts, true);
+ view.setInt32(40, flatPoints.length, true);
+ let offset = 44;
+ let runningIndex = 0;
+ for (const part of parts) {
+ view.setInt32(offset, runningIndex, true);
+ offset += 4;
+ runningIndex += part.length;
+ }
+ for (const [x, y] of flatPoints) {
+ view.setFloat64(offset, Number(x), true);
+ view.setFloat64(offset + 8, Number(y), true);
+ offset += 16;
+ }
+ return content;
+}
+
+/** Write the 100-byte .shp/.shx header. `fileWords` is the file length in
+ * 16-bit words; `box` the dataset bounding box. */
+function writeMainHeader(target: Uint8Array, shapeType: number, fileWords: number, box: Box): void {
+ const view = new DataView(target.buffer, target.byteOffset, 100);
+ view.setInt32(0, 9994, false); // file code (big-endian)
+ view.setInt32(24, fileWords, false); // file length in words (big-endian)
+ view.setInt32(28, 1000, true); // version (little-endian)
+ view.setInt32(32, shapeType, true);
+ const drawn = safeBox(box);
+ view.setFloat64(36, drawn.minX, true);
+ view.setFloat64(44, drawn.minY, true);
+ view.setFloat64(52, drawn.maxX, true);
+ view.setFloat64(60, drawn.maxY, true);
+ // Z and M ranges remain zero (2D output).
+}
+
+// ----- DBF (attribute table) -----
+
+type DbfFieldType = 'C' | 'N' | 'L';
+
+interface DbfField {
+ key: string;
+ name: string;
+ type: DbfFieldType;
+ length: number;
+ decimals: number;
+}
+
+const TEXT_ENCODER = new TextEncoder();
+
+/** Encode `text` to UTF-8, truncated to at most `maxBytes` without splitting a
+ * multi-byte code point (so non-ASCII attribute values stay valid). */
+function encodeUtf8Truncated(text: string, maxBytes: number): Uint8Array {
+ const full = TEXT_ENCODER.encode(text);
+ if (full.length <= maxBytes) return full;
+ let length = 0;
+ for (const char of text) {
+ const size = TEXT_ENCODER.encode(char).length;
+ if (length + size > maxBytes) break;
+ length += size;
+ }
+ return full.subarray(0, length);
+}
+
+function dbfFieldName(key: string, taken: Set<string>): string {
+ // DBF field names are at most 10 characters; truncate and de-duplicate.
+ const base = key.replace(/[^0-9A-Za-z_]/g, '_').slice(0, 10) || 'field';
+ if (!taken.has(base.toUpperCase())) {
+ taken.add(base.toUpperCase());
+ return base;
+ }
+ for (let i = 1; i < 1000; i += 1) {
+ const suffix = String(i);
+ const candidate = `${base.slice(0, 10 - suffix.length)}${suffix}`;
+ if (!taken.has(candidate.toUpperCase())) {
+ taken.add(candidate.toUpperCase());
+ return candidate;
+ }
+ }
+ // Falling through means >999 fields share one 10-char prefix; returning a
+ // name already in `taken` would write a duplicate DBF column. Fail loudly
+ // instead of silently corrupting the file.
+ throw new Error(`Cannot generate a unique DBF field name for "${key}" (>999 collisions).`);
+}
+
+function decimalPlaces(value: number): number {
+ if (Number.isInteger(value)) return 0;
+ const text = String(value);
+ const dot = text.indexOf('.');
+ if (dot < 0) return 0;
+ return Math.min(15, text.length - dot - 1);
+}
+
+function planField(key: string, name: string, values: unknown[]): DbfField {
+ let sawNumber = false;
+ let sawNonNumber = false;
+ let sawBoolean = false;
+ let sawNonBoolean = false;
+ for (const value of values) {
+ if (value == null) continue;
+ if (typeof value === 'boolean') {
+ sawBoolean = true;
+ } else {
+ sawNonBoolean = true;
+ }
+ if (typeof value === 'number' && Number.isFinite(value)) {
+ sawNumber = true;
+ } else {
+ sawNonNumber = true;
+ }
+ }
+
+ if (sawBoolean && !sawNonBoolean) {
+ return {key, name, type: 'L', length: 1, decimals: 0};
+ }
+
+ if (sawNumber && !sawNonNumber) {
+ let decimals = 0;
+ for (const value of values) {
+ if (typeof value === 'number' && Number.isFinite(value)) {
+ decimals = Math.max(decimals, decimalPlaces(value));
+ }
+ }
+ let length = 1;
+ for (const value of values) {
+ if (typeof value !== 'number' || !Number.isFinite(value)) continue;
+ const text = decimals > 0 ? value.toFixed(decimals) : String(value);
+ length = Math.max(length, text.length);
+ }
+ return {key, name, type: 'N', length: Math.min(length, 33), decimals};
+ }
+
+ // Fall back to text for everything else (strings, mixed types, objects).
+ let length = 1;
+ for (const value of values) {
+ if (value == null) continue;
+ const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
+ length = Math.max(length, TEXT_ENCODER.encode(text).length);
+ }
+ return {key, name, type: 'C', length: Math.min(length, 254), decimals: 0};
+}
+
+function encodeDbfValue(value: unknown, field: DbfField): Uint8Array {
+ const cell = new Uint8Array(field.length).fill(0x20); // space-padded
+ if (value == null) return cell;
+
+ if (field.type === 'L') {
+ cell[0] = value ? 0x54 : 0x46; // 'T' / 'F'
+ return cell;
+ }
+
+ if (field.type === 'N') {
+ if (typeof value !== 'number' || !Number.isFinite(value)) return cell;
+ let text = field.decimals > 0 ? value.toFixed(field.decimals) : String(value);
+ if (text.length > field.length) text = text.slice(0, field.length);
+ // Numbers are right-justified in DBF.
+ const bytes = TEXT_ENCODER.encode(text);
+ cell.set(bytes, field.length - bytes.length);
+ return cell;
+ }
+
+ // Text: left-justified, truncated to the field byte length on a code-point
+ // boundary so multi-byte characters are never cut mid-sequence.
+ const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
+ cell.set(encodeUtf8Truncated(text, field.length), 0);
+ return cell;
+}
+
+function buildDbf(features: Feature[], fields: DbfField[]): Uint8Array {
+ const recordLength = 1 + fields.reduce((total, field) => total + field.length, 0);
+ const headerLength = 32 + fields.length * 32 + 1;
+ // Both are serialized as uint16; too many or too-wide attributes would
+ // overflow and silently corrupt the .dbf, so fail fast instead.
+ if (recordLength > 0xffff || headerLength > 0xffff) {
+ throw new Error('Too many or too-wide attributes for the DBF format limits.');
+ }
+ const total = headerLength + features.length * recordLength + 1; // +1 EOF
+ const buffer = new Uint8Array(total);
+ const view = new DataView(buffer.buffer);
+
+ buffer[0] = 0x03; // dBASE III without memo
+ const now = new Date();
+ buffer[1] = now.getFullYear() - 1900;
+ buffer[2] = now.getMonth() + 1;
+ buffer[3] = now.getDate();
+ view.setUint32(4, features.length, true);
+ view.setUint16(8, headerLength, true);
+ view.setUint16(10, recordLength, true);
+
+ let offset = 32;
+ for (const field of fields) {
+ const nameBytes = TEXT_ENCODER.encode(field.name).subarray(0, 11);
+ buffer.set(nameBytes, offset); // remaining name bytes stay 0 (null-padded)
+ buffer[offset + 11] = field.type.charCodeAt(0);
+ buffer[offset + 16] = field.length;
+ buffer[offset + 17] = field.decimals;
+ offset += 32;
+ }
+ buffer[offset] = 0x0d; // field descriptor terminator
+ offset += 1;
+
+ for (const feature of features) {
+ buffer[offset] = 0x20; // record not deleted
+ offset += 1;
+ for (const field of fields) {
+ const cell = encodeDbfValue(feature.properties?.[field.key], field);
+ buffer.set(cell, offset);
+ offset += field.length;
+ }
+ }
+ buffer[offset] = 0x1a; // end-of-file marker
+ return buffer;
+}
+
+/**
+ * Assemble the parts of an ESRI Shapefile from a FeatureCollection.
+ *
+ * Args:
+ * geojson: Features to write (assumed WGS84).
+ *
+ * Returns:
+ * The .shp/.shx/.dbf/.prj/.cpg byte buffers (the caller names and zips them).
+ */
+export function writeShapefile(geojson: FeatureCollection): ShapefileParts {
+ const features = geojson.features ?? [];
+ if (features.length === 0) {
+ throw new Error('The layer has no features to export.');
+ }
+ const {type: shapeType, family} = fileShapeType(features);
+
+ const fileBox = emptyBox();
+ const contents = features.map(feature =>
+ encodeShape(feature.geometry, shapeType, family, fileBox)
+ );
+
+ // .shp: 100-byte header + per-record (8-byte header + content).
+ const shpSize = 100 + contents.reduce((total, content) => total + 8 + content.length, 0);
+ const shp = new Uint8Array(shpSize);
+ const shpView = new DataView(shp.buffer);
+ const shxSize = 100 + contents.length * 8;
+ const shx = new Uint8Array(shxSize);
+ const shxView = new DataView(shx.buffer);
+
+ writeMainHeader(shp, shapeType, shpSize / 2, fileBox);
+ writeMainHeader(shx, shapeType, shxSize / 2, fileBox);
+
+ let shpOffset = 100;
+ let shxOffset = 100;
+ contents.forEach((content, index) => {
+ const contentWords = content.length / 2;
+ const recordOffsetWords = shpOffset / 2;
+ // Record header (big-endian): 1-based record number + content length.
+ shpView.setInt32(shpOffset, index + 1, false);
+ shpView.setInt32(shpOffset + 4, contentWords, false);
+ shp.set(content, shpOffset + 8);
+ shpOffset += 8 + content.length;
+ // .shx entry (big-endian): record offset + content length, both in words.
+ shxView.setInt32(shxOffset, recordOffsetWords, false);
+ shxView.setInt32(shxOffset + 4, contentWords, false);
+ shxOffset += 8;
+ });
+
+ // DBF fields from the union of property keys across all features.
+ const keys: string[] = [];
+ const seen = new Set<string>();
+ for (const feature of features) {
+ for (const key of Object.keys(feature.properties ?? {})) {
+ if (!seen.has(key)) {
+ seen.add(key);
+ keys.push(key);
+ }
+ }
+ }
+ const takenNames = new Set<string>();
+ const fields = keys.map(key =>
+ planField(
+ key,
+ dbfFieldName(key, takenNames),
+ features.map(feature => feature.properties?.[key])
+ )
+ );
+ const dbf = buildDbf(features, fields);
+
+ return {
+ shp,
+ shx,
+ dbf,
+ prj: TEXT_ENCODER.encode(WGS84_PRJ),
+ cpg: TEXT_ENCODER.encode('UTF-8')
+ };
+}
diff --git a/src/processors/src/export/sql-js.spec.ts b/src/processors/src/export/sql-js.spec.ts
new file mode 100644
index 000000000..9437ed841
--- /dev/null
+++ b/src/processors/src/export/sql-js.spec.ts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {loadSqlJs} from './sql-js';
+
+describe('loadSqlJs', () => {
+ test('uses the package loader in a production Node environment', async () => {
+ const previousNodeEnv = process.env.NODE_ENV;
+ const documentDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'document');
+ process.env.NODE_ENV = 'production';
+ Object.defineProperty(globalThis, 'document', {configurable: true, value: undefined});
+ try {
+ expect(typeof document).toBe('undefined');
+ const SQL = await loadSqlJs();
+ const db = new SQL.Database();
+ db.run('CREATE TABLE places (name TEXT);');
+ expect(db.exec("SELECT name FROM sqlite_master WHERE name = 'places'")[0].values).toEqual([
+ ['places']
+ ]);
+ db.close();
+ } finally {
+ process.env.NODE_ENV = previousNodeEnv;
+ if (documentDescriptor) {
+ Object.defineProperty(globalThis, 'document', documentDescriptor);
+ }
+ }
+ });
+});
diff --git a/src/processors/src/export/sql-js.ts b/src/processors/src/export/sql-js.ts
new file mode 100644
index 000000000..8e0b16ca2
--- /dev/null
+++ b/src/processors/src/export/sql-js.ts
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import type {SqlJsStatic} from 'sql.js';
+
+/** Keep in sync with the sql.js dependency in package.json. */
+const SQL_JS_VERSION = '1.13.0';
+const SQL_JS_CDN = `https://cdn.jsdelivr.net/npm/sql.js@${SQL_JS_VERSION}/dist/`;
+
+let sqlPromise: Promise<SqlJsStatic> | null = null;
+
+type InitSqlJs = (config?: {locateFile?: (file: string) => string}) => Promise<SqlJsStatic>;
+
+/**
+ * Load sql.js once.
+ * Tests import the package, which reads the wasm from disk. The browser build
+ * of that package calls Node's fs/path/crypto, so the demo bundle loads the
+ * script from a CDN instead of importing it.
+ */
+export function loadSqlJs(): Promise<SqlJsStatic> {
+ if (!sqlPromise) {
+ const pending = (
+ process.env.NODE_ENV === 'test' || typeof document === 'undefined'
+ ? loadSqlJsFromPackage()
+ : loadSqlJsFromCdn()
+ ).catch(error => {
+ sqlPromise = null;
+ throw error;
+ });
+ sqlPromise = pending;
+ }
+ return sqlPromise;
+}
+
+async function loadSqlJsFromPackage(): Promise<SqlJsStatic> {
+ // Non-literal so browser bundlers do not pull in sql-wasm.js (it requires fs).
+ const specifier = 'sql.js';
+ const sqlModule = (await import(specifier)) as {default: InitSqlJs};
+ return sqlModule.default();
+}
+
+function loadSqlJsFromCdn(): Promise<SqlJsStatic> {
+ const locateFile = (file: string) => `${SQL_JS_CDN}${file}`;
+ const existing = (globalThis as {initSqlJs?: InitSqlJs}).initSqlJs;
+ if (existing) {
+ return existing({locateFile});
+ }
+ return new Promise((resolve, reject) => {
+ const script = document.createElement('script');
+ script.src = `${SQL_JS_CDN}sql-wasm.js`;
+ script.async = true;
+ script.onload = () => {
+ const init = (globalThis as {initSqlJs?: InitSqlJs}).initSqlJs;
+ if (!init) {
+ reject(new Error('sql.js failed to initialize'));
+ return;
+ }
+ resolve(init({locateFile}));
+ };
+ script.onerror = () => reject(new Error('Failed to load sql.js'));
+ document.head.appendChild(script);
+ });
+}
diff --git a/src/processors/src/export/vector-export.spec.ts b/src/processors/src/export/vector-export.spec.ts
new file mode 100644
index 000000000..106fa6827
--- /dev/null
+++ b/src/processors/src/export/vector-export.spec.ts
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {convertArrowToSchema} from '@loaders.gl/schema-utils';
+import {ALL_FIELD_TYPES, EXPORT_DATA_TYPE} from '@kepler.gl/constants';
+import {formatCsv} from '@kepler.gl/reducers';
+import {Field} from '@kepler.gl/types';
+import {createDataContainer} from '@kepler.gl/utils';
+import {tableFromIPC} from 'apache-arrow';
+
+import {datasetToFeatureCollection, encodeDatasetExport} from './vector-export';
+import {loadSqlJs} from './sql-js';
+
+const fields = [
+ {name: 'name', type: ALL_FIELD_TYPES.string},
+ {name: 'secret', type: ALL_FIELD_TYPES.string},
+ {name: 'lat', type: ALL_FIELD_TYPES.real},
+ {name: 'lng', type: ALL_FIELD_TYPES.real}
+] as Field[];
+
+const rows = [
+ ['Harbor', 'hidden-value', 37.8, -122.4],
+ ['Pier', 'also-hidden', 37.81, -122.41]
+];
+
+const fieldPairs = [
+ {
+ defaultName: 'point',
+ pair: {
+ lat: {fieldIdx: 2, value: 'lat'},
+ lng: {fieldIdx: 3, value: 'lng'}
+ },
+ suffix: ['lat', 'lng']
+ }
+];
+
+function container() {
+ return createDataContainer(rows);
+}
+
+describe('dataset export', () => {
+ test('omits columns hidden in the attribute table', () => {
+ const collection = datasetToFeatureCollection(container(), fields, {
+ hiddenColumns: ['secret'],
+ fieldPairs
+ });
+
+ expect(collection.features).toHaveLength(2);
+ expect(collection.features[0].geometry).toEqual({
+ type: 'Point',
+ coordinates: [-122.4, 37.8]
+ });
+ expect(collection.features[0].properties).toEqual({
+ name: 'Harbor',
+ lat: 37.8,
+ lng: -122.4
+ });
+ expect(collection.features[0].properties).not.toHaveProperty('secret');
+
+ const csv = formatCsv(container(), fields, ['secret']);
+ expect(csv).toContain('Harbor');
+ expect(csv).not.toContain('hidden-value');
+ expect(csv).not.toContain('secret');
+ });
+
+ test('writes geojson, kml, kmz, shapefile, geopackage, and geoparquet', async () => {
+ const shared = {
+ data: container(),
+ fields,
+ fieldPairs,
+ hiddenColumns: ['secret'],
+ layerName: 'places'
+ };
+
+ const geojson = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.GEOJSON});
+ const parsed = JSON.parse(String(geojson.data));
+ expect(parsed.features[1].properties.name).toBe('Pier');
+ expect(parsed.features[1].properties.secret).toBeUndefined();
+ expect(geojson.extension).toBe('geojson');
+
+ const kml = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.KML});
+ expect(String(kml.data)).toContain('<coordinates>-122.4,37.8</coordinates>');
+ expect(String(kml.data)).not.toContain('hidden-value');
+
+ const kmz = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.KMZ});
+ expect(Buffer.from(kmz.data as Uint8Array).includes(Buffer.from('doc.kml'))).toBe(true);
+
+ const shapefile = await encodeDatasetExport({
+ ...shared,
+ dataType: EXPORT_DATA_TYPE.SHAPEFILE
+ });
+ const shapefileBytes = Buffer.from(shapefile.data as Uint8Array);
+ expect(shapefile.extension).toBe('zip');
+ expect(shapefileBytes.includes(Buffer.from('places.shp'))).toBe(true);
+ expect(shapefileBytes.includes(Buffer.from('places.dbf'))).toBe(true);
+ expect(shapefileBytes.includes(Buffer.from('hidden-value'))).toBe(false);
+
+ const geopackage = await encodeDatasetExport({
+ ...shared,
+ dataType: EXPORT_DATA_TYPE.GEOPACKAGE
+ });
+ const SQL = await loadSqlJs();
+ const db = new SQL.Database(new Uint8Array(geopackage.data as Uint8Array));
+ const exported = db.exec('SELECT name FROM places');
+ expect(exported[0].values.map(row => row[0])).toEqual(['Harbor', 'Pier']);
+ const columns = db.exec('PRAGMA table_info(places)');
+ const columnNames = columns[0].values.map(row => row[1]);
+ expect(columnNames).not.toContain('secret');
+ db.close();
+
+ const geoparquet = await encodeDatasetExport({
+ ...shared,
+ dataType: EXPORT_DATA_TYPE.GEOPARQUET
+ });
+ const parquetBytes = Buffer.from(geoparquet.data as Uint8Array);
+ expect(parquetBytes.subarray(0, 4).toString()).toBe('PAR1');
+ expect(parquetBytes.includes(Buffer.from('"encoding":"WKB"'))).toBe(true);
+ expect(parquetBytes.includes(Buffer.from('hidden-value'))).toBe(false);
+ });
+
+ test('rejects mixed geometry families when exporting a shapefile', async () => {
+ const geometryFields = [
+ {name: '_geojson', type: ALL_FIELD_TYPES.geojson},
+ {name: 'label', type: ALL_FIELD_TYPES.string}
+ ] as Field[];
+ const geometryRows = [
+ [{type: 'Point', coordinates: [0, 0]}, 'point'],
+ [
+ {
+ type: 'LineString',
+ coordinates: [
+ [0, 0],
+ [1, 1]
+ ]
+ },
+ 'line'
+ ]
+ ];
+
+ await expect(
+ encodeDatasetExport({
+ data: createDataContainer(geometryRows),
+ fields: geometryFields,
+ dataType: EXPORT_DATA_TYPE.SHAPEFILE,
+ layerName: 'mixed'
+ })
+ ).rejects.toThrow('Shapefile export requires all geometries to have the same shape family.');
+ });
+
+ test('writes geoparquet string columns the importer can read', async () => {
+ const geoparquet = await encodeDatasetExport({
+ data: container(),
+ fields,
+ fieldPairs,
+ dataType: EXPORT_DATA_TYPE.GEOPARQUET,
+ layerName: 'places'
+ });
+ const specifier = 'parquet-wasm/node';
+ const imported = (await import(specifier)) as {
+ readParquet?: (bytes: Uint8Array) => {intoIPCStream(): Uint8Array};
+ default?: {readParquet: (bytes: Uint8Array) => {intoIPCStream(): Uint8Array}};
+ };
+ const api = imported.readParquet ? imported : imported.default;
+ if (!api?.readParquet) {
+ throw new Error('parquet-wasm failed to load');
+ }
+ const wasmTable = api.readParquet(new Uint8Array(geoparquet.data as Uint8Array));
+ const arrowTable = tableFromIPC(wasmTable.intoIPCStream());
+
+ expect(arrowTable.getChild('name')?.toArray()).toEqual(['Harbor', 'Pier']);
+ expect(arrowTable.getChild('geometry')?.get(0)).toBeInstanceOf(Uint8Array);
+ const schema = convertArrowToSchema(arrowTable.schema);
+ expect(schema.fields.find(field => field.name === 'geometry')?.type).toBe('binary');
+ expect(schema.fields.find(field => field.name === 'name')?.type).toBe('utf8');
+ });
+
+ test.each(['gpkg_contents', 'gpkg_geometry_columns', 'sqlite_sequence'])(
+ 'sanitizes the reserved GeoPackage layer name %s',
+ async layerName => {
+ const geopackage = await encodeDatasetExport({
+ data: container(),
+ fields,
+ fieldPairs,
+ dataType: EXPORT_DATA_TYPE.GEOPACKAGE,
+ layerName
+ });
+ const SQL = await loadSqlJs();
+ const db = new SQL.Database(new Uint8Array(geopackage.data as Uint8Array));
+ try {
+ const contents = db.exec('SELECT table_name FROM gpkg_contents');
+ const exportedTable = String(contents[0].values[0][0]);
+ expect(exportedTable).not.toBe(layerName);
+ expect(exportedTable).not.toMatch(/^sqlite_/i);
+ expect(db.exec(`SELECT COUNT(*) FROM "${exportedTable}"`)[0].values[0][0]).toBe(2);
+ } finally {
+ db.close();
+ }
+ }
+ );
+
+ test('uses a geojson column as feature geometry and drops it from properties', async () => {
+ const polygon = {
+ type: 'Polygon',
+ coordinates: [
+ [
+ [0, 0],
+ [1, 0],
+ [1, 1],
+ [0, 1],
+ [0, 0]
+ ]
+ ]
+ };
+ const geoFields = [
+ {name: '_geojson', type: ALL_FIELD_TYPES.geojson},
+ {name: 'label', type: ALL_FIELD_TYPES.string}
+ ] as Field[];
+ const geoRows = [[polygon, 'block']];
+ const collection = datasetToFeatureCollection(createDataContainer(geoRows), geoFields);
+
+ expect(collection.features[0].geometry).toEqual(polygon);
+ expect(collection.features[0].properties).toEqual({label: 'block'});
+ });
+});
diff --git a/src/processors/src/export/vector-export.ts b/src/processors/src/export/vector-export.ts
new file mode 100644
index 000000000..87e2e0490
--- /dev/null
+++ b/src/processors/src/export/vector-export.ts
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {EXPORT_DATA_TYPE} from '@kepler.gl/constants';
+import {Field, FieldPair} from '@kepler.gl/types';
+import {DataContainerInterface} from '@kepler.gl/utils';
+import {ZipWriter} from '@loaders.gl/zip';
+import type {FeatureCollection} from 'geojson';
+
+import {datasetToFeatureCollection} from './dataset-to-geojson';
+import {writeGeoPackage} from './geopackage-writer';
+import {writeGeoParquet} from './geoparquet-writer';
+import {writeKml} from './kml-writer';
+import {writeShapefile} from './shapefile-writer';
+
+export {datasetToFeatureCollection} from './dataset-to-geojson';
+
+export type EncodedDatasetExport = {
+ data: BlobPart;
+ extension: string;
+ mimeType: string;
+};
+
+/** Turn a dataset label into a filesystem-safe export base name. */
+export function sanitizeExportName(name: string): string {
+ let sanitized = '';
+ for (const char of name.trim()) {
+ const code = char.charCodeAt(0);
+ sanitized += code <= 0x1f || '<>:"/\\|?*'.includes(char) ? '-' : char;
+ }
+ sanitized = sanitized.replace(/\s+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
+ return sanitized || 'dataset';
+}
+
+function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
+ const copy = new Uint8Array(bytes.byteLength);
+ copy.set(bytes);
+ return copy.buffer;
+}
+
+async function zipEntries(entries: Record<string, Uint8Array>): Promise<Uint8Array> {
+ const files = Object.fromEntries(
+ Object.entries(entries).map(([name, bytes]) => [name, toArrayBuffer(bytes)])
+ );
+ return new Uint8Array(await ZipWriter.encode(files));
+}
+
+/**
+ * Encode a dataset for download. Hidden attribute-table columns are omitted.
+ * Geometry columns that are hidden are still used to build the feature geometry.
+ */
+export async function encodeDatasetExport(options: {
+ data: DataContainerInterface;
+ fields: Field[];
+ fieldPairs?: FieldPair[];
+ hiddenColumns?: string[];
+ dataType: string;
+ layerName: string;
+}): Promise<EncodedDatasetExport> {
+ const layerName = sanitizeExportName(options.layerName);
+ const geojson = datasetToFeatureCollection(options.data, options.fields, {
+ hiddenColumns: options.hiddenColumns,
+ fieldPairs: options.fieldPairs
+ });
+
+ switch (options.dataType) {
+ case EXPORT_DATA_TYPE.GEOJSON:
+ return {
+ data: JSON.stringify(geojson),
+ extension: 'geojson',
+ mimeType: 'application/geo+json'
+ };
+ case EXPORT_DATA_TYPE.KML:
+ return {
+ data: writeKml(geojson, layerName),
+ extension: 'kml',
+ mimeType: 'application/vnd.google-earth.kml+xml'
+ };
+ case EXPORT_DATA_TYPE.KMZ:
+ return {
+ data: await zipEntries({
+ 'doc.kml': new TextEncoder().encode(writeKml(geojson, layerName))
+ }),
+ extension: 'kmz',
+ mimeType: 'application/vnd.google-earth.kmz'
+ };
+ case EXPORT_DATA_TYPE.SHAPEFILE:
+ return {
+ data: await zipShapefile(geojson, layerName),
+ extension: 'zip',
+ mimeType: 'application/zip'
+ };
+ case EXPORT_DATA_TYPE.GEOPACKAGE:
+ return {
+ data: await writeGeoPackage(geojson, layerName),
+ extension: 'gpkg',
+ mimeType: 'application/geopackage+sqlite3'
+ };
+ case EXPORT_DATA_TYPE.GEOPARQUET:
+ return {
+ data: await writeGeoParquet(geojson),
+ extension: 'parquet',
+ mimeType: 'application/vnd.apache.parquet'
+ };
+ default:
+ throw new Error(`Unsupported export format ${options.dataType}`);
+ }
+}
+
+async function zipShapefile(geojson: FeatureCollection, baseName: string): Promise<Uint8Array> {
+ const parts = writeShapefile(geojson);
+ return zipEntries({
+ [`${baseName}.shp`]: parts.shp,
+ [`${baseName}.shx`]: parts.shx,
+ [`${baseName}.dbf`]: parts.dbf,
+ [`${baseName}.prj`]: parts.prj,
+ [`${baseName}.cpg`]: parts.cpg
+ });
+}
diff --git a/src/processors/src/index.ts b/src/processors/src/index.ts
index 2b24f7f64..c921aaa44 100644
--- a/src/processors/src/index.ts
+++ b/src/processors/src/index.ts
@@ -8,3 +8,9 @@ export * from './types';
export * from './kepler-csv-loader';
export * from './loader-registry';
export * from './shapefile-files';
+export {
+ datasetToFeatureCollection,
+ encodeDatasetExport,
+ sanitizeExportName
+} from './export/vector-export';
+export type {EncodedDatasetExport} from './export/vector-export';
diff --git a/src/reducers/src/column-edit-updaters.spec.ts b/src/reducers/src/column-edit-updaters.spec.ts
index 0c22cb985..67c6e131c 100644
--- a/src/reducers/src/column-edit-updaters.spec.ts
+++ b/src/reducers/src/column-edit-updaters.spec.ts
@@ -95,6 +95,7 @@ function withCities(): VisState {
describe('column edits', () => {
test('rename changes the field identity stored in layers, filters, and saved maps', () => {
let state = withCities();
+ state.datasets.cities.hiddenColumns = ['region', 'pop'];
expect(state.layers).toHaveLength(1);
expect(state.filters[0].name).toEqual(['region']);
@@ -122,6 +123,7 @@ describe('column edits', () => {
expect(state.charts[0].yAxis?.field?.name).toBe('pop');
expect(state.charts[0].crossFilter?.fieldNames).toEqual({xAxis: 'area'});
expect(state.interactionConfig.tooltip.config.fieldsToShow.cities[0].name).toBe('area');
+ expect(state.datasets.cities.hiddenColumns).toEqual(['area', 'pop']);
const saved = SchemaManager.save({
visState: state,
@@ -143,12 +145,14 @@ describe('column edits', () => {
test('delete drops references and refuses the last column', () => {
let state = withCities();
+ state.datasets.cities.hiddenColumns = ['region', 'pop'];
state = reduceVisState(state, VisStateActions.deleteTableColumn('cities', 'pop'));
expect(state.datasets.cities.fields.map(field => field.name)).toEqual(['lat', 'lng', 'region']);
expect(state.layers).toHaveLength(1);
expect(state.filters).toHaveLength(1);
expect(state.charts[0].yAxis?.field).toBeNull();
+ expect(state.datasets.cities.hiddenColumns).toEqual(['region']);
state = reduceVisState(state, VisStateActions.deleteTableColumn('cities', 'lat'));
expect(state.layers).toHaveLength(0);
diff --git a/src/reducers/src/export-utils.ts b/src/reducers/src/export-utils.ts
index f8a79eed3..60b70df41 100644
--- a/src/reducers/src/export-utils.ts
+++ b/src/reducers/src/export-utils.ts
@@ -5,6 +5,7 @@ import {Blob} from 'global/window';
import {csvFormatRows} from 'd3-dsv';
import {EXPORT_DATA_TYPE} from '@kepler.gl/constants';
+import {encodeDatasetExport, sanitizeExportName} from '@kepler.gl/processors';
import {Field} from '@kepler.gl/types';
import KeplerTable, {Datasets} from '@kepler.gl/table';
@@ -21,7 +22,7 @@ interface StateType {
appName?: string;
}
-export function exportData(state: StateType, options) {
+export async function exportData(state: StateType, options): Promise<void> {
const {visState, appName} = state;
const {datasets} = visState;
const {selectedDataset, dataType, filtered} = options;
@@ -35,26 +36,48 @@ export function exportData(state: StateType, options) {
return;
}
- selectedDatasets.forEach(selectedData => {
- const {dataContainer, fields, label, filteredIdxCPU = []} = selectedData as KeplerTable;
- const toExport = filtered
- ? createIndexedDataContainer(dataContainer, filteredIdxCPU)
- : dataContainer;
+ const errors: unknown[] = [];
+ for (const selectedData of selectedDatasets) {
+ try {
+ const {
+ dataContainer,
+ fields,
+ label,
+ filteredIdxCPU = [],
+ hiddenColumns,
+ fieldPairs
+ } = selectedData as KeplerTable;
+ const toExport = filtered
+ ? createIndexedDataContainer(dataContainer, filteredIdxCPU)
+ : dataContainer;
+ const layerName = sanitizeExportName(label);
- // start to export data according to selected data type
- switch (dataType) {
- case EXPORT_DATA_TYPE.CSV: {
- const csv = formatCsv(toExport, fields);
-
- const fileBlob = new Blob([csv], {type: 'text/csv'});
- downloadFile(fileBlob, `${filename}_${label}.csv`);
- break;
+ if (dataType === EXPORT_DATA_TYPE.CSV) {
+ const csv = formatCsv(toExport, fields, hiddenColumns);
+ downloadFile(new Blob([csv], {type: 'text/csv'}), `${filename}_${layerName}.csv`);
+ continue;
}
- // TODO: support more file types.
- default:
- break;
+
+ const file = await encodeDatasetExport({
+ data: toExport,
+ fields,
+ fieldPairs,
+ hiddenColumns,
+ dataType,
+ layerName
+ });
+ downloadFile(
+ new Blob([file.data], {type: file.mimeType}),
+ `${filename}_${layerName}.${file.extension}`
+ );
+ } catch (error) {
+ errors.push(error);
}
- });
+ }
+
+ if (errors.length) {
+ throw errors[0];
+ }
}
/**
@@ -63,13 +86,29 @@ export function exportData(state: StateType, options) {
* @param fields `dataset.fields`
* @returns csv string
*/
-export function formatCsv(data: DataContainerInterface, fields: Field[]): string {
- const columns = fields.map(f => f.displayName || f.name);
+export function formatCsv(
+ data: DataContainerInterface,
+ fields: Field[],
+ hiddenColumns: string[] = []
+): string {
+ const hidden = new Set(hiddenColumns);
+ const included = fields
+ .map((field, index) => ({field, index}))
+ .filter(({field}) => !hidden.has(field.name));
+ const columns = included.map(({field}) => field.displayName || field.name);
const formattedData = [columns];
// parse geojson object as string
for (const row of data.rows(true)) {
- formattedData.push(row.map((d, i) => parseFieldValue(d, fields[i].type, fields[i])));
+ formattedData.push(
+ included.map(({field, index}) =>
+ parseFieldValue(
+ typeof row.valueAt === 'function' ? row.valueAt(index) : row[index],
+ field.type,
+ field
+ )
+ )
+ );
}
return csvFormatRows(formattedData);
diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts
index 62f856eaa..e298870b5 100644
--- a/src/reducers/src/vis-state-updaters.ts
+++ b/src/reducers/src/vis-state-updaters.ts
@@ -196,6 +196,8 @@ import {
copyTableAndUpdate,
createNewDataEntry,
pinTableColumns,
+ showAllTableColumns,
+ toggleHiddenTableColumn,
setFilterGpuMode,
sortDatasetByColumn
} from '@kepler.gl/table';
@@ -4377,11 +4379,14 @@ export function updateDatasetUpdater(
newTable.filterRecordCPU = undefined;
newTable.sortColumn = undefined;
newTable.sortOrder = null;
+ const newFieldsByName = new Set(newTable.fields.map(f => f.name));
newTable.pinnedColumns = (existing.pinnedColumns || [])
.map(name => renames[name] ?? name)
- .filter(name => newTable.getColumnFieldIdx(name) > -1);
+ .filter(name => newFieldsByName.has(name));
+ newTable.hiddenColumns = (existing.hiddenColumns || [])
+ .map(name => renames[name] ?? name)
+ .filter(name => newFieldsByName.has(name));
- const newFieldsByName = new Set(newTable.fields.map(f => f.name));
const newDatasets = {...state.datasets, [dataId]: newTable};
// 2. reconcile filters
@@ -5625,6 +5630,36 @@ export function pinTableColumnUpdater(
return set(['datasets', dataId], newDataset, state);
}
+/**
+ * @memberof visStateUpdaters
+ * @public
+ */
+export function toggleTableColumnHiddenUpdater(
+ state: VisState,
+ {dataId, column}: VisStateActions.ToggleTableColumnHiddenUpdaterAction
+): VisState {
+ const dataset = state.datasets[dataId];
+ if (!dataset) {
+ return state;
+ }
+ return set(['datasets', dataId], toggleHiddenTableColumn(dataset, column), state);
+}
+
+/**
+ * @memberof visStateUpdaters
+ * @public
+ */
+export function showAllTableColumnsUpdater(
+ state: VisState,
+ {dataId}: VisStateActions.ShowAllTableColumnsUpdaterAction
+): VisState {
+ const dataset = state.datasets[dataId];
+ if (!dataset) {
+ return state;
+ }
+ return set(['datasets', dataId], showAllTableColumns(dataset), state);
+}
+
/**
* Copy column content as strings
* @memberof visStateUpdaters
diff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.ts
index b839e5549..027e18fbf 100644
--- a/src/reducers/src/vis-state.ts
+++ b/src/reducers/src/vis-state.ts
@@ -178,6 +178,10 @@ const actionHandler = {
[ActionTypes.PIN_TABLE_COLUMN]: visStateUpdaters.pinTableColumnUpdater,
+ [ActionTypes.TOGGLE_TABLE_COLUMN_HIDDEN]: visStateUpdaters.toggleTableColumnHiddenUpdater,
+
+ [ActionTypes.SHOW_ALL_TABLE_COLUMNS]: visStateUpdaters.showAllTableColumnsUpdater,
+
[ActionTypes.COPY_TABLE_COLUMN]: visStateUpdaters.copyTableColumnUpdater,
[ActionTypes.SET_COLUMN_DISPLAY_FORMAT]: visStateUpdaters.setColumnDisplayFormatUpdater,
diff --git a/src/table/src/hidden-columns.spec.ts b/src/table/src/hidden-columns.spec.ts
new file mode 100644
index 000000000..10d8fbca7
--- /dev/null
+++ b/src/table/src/hidden-columns.spec.ts
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+
+import KeplerTable, {showAllTableColumns, toggleHiddenTableColumn} from './kepler-table';
+
+function makeTable() {
+ const table = new KeplerTable({
+ info: {id: 'places', label: 'places'},
+ color: [0, 0, 0]
+ });
+ table.updateSchema({
+ fields: [
+ {name: 'name', type: ALL_FIELD_TYPES.string, analyzerType: ALL_FIELD_TYPES.string},
+ {name: 'secret', type: ALL_FIELD_TYPES.string, analyzerType: ALL_FIELD_TYPES.string}
+ ],
+ rows: [['Harbor', 'hidden-value']]
+ });
+ return table;
+}
+
+describe('hidden table columns', () => {
+ test('toggle hides a column and show all restores it', () => {
+ const hidden = toggleHiddenTableColumn(makeTable(), 'secret');
+ expect(hidden.hiddenColumns).toEqual(['secret']);
+
+ const shown = toggleHiddenTableColumn(hidden, 'secret');
+ expect(shown.hiddenColumns).toEqual([]);
+
+ const both = toggleHiddenTableColumn(toggleHiddenTableColumn(makeTable(), 'secret'), 'name');
+ expect(showAllTableColumns(both).hiddenColumns).toEqual([]);
+ });
+
+ test('ignores a column that is not on the dataset', () => {
+ const table = makeTable();
+ expect(toggleHiddenTableColumn(table, 'missing')).toBe(table);
+ });
+});
diff --git a/src/table/src/index.ts b/src/table/src/index.ts
index 87e2aee87..1869a7be4 100644
--- a/src/table/src/index.ts
+++ b/src/table/src/index.ts
@@ -7,6 +7,8 @@ export {
findPointFieldPairs,
copyTableAndUpdate,
pinTableColumns,
+ showAllTableColumns,
+ toggleHiddenTableColumn,
sortDatasetByColumn,
copyTable,
maybeToDate
diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts
index 351b79aa1..8295f918f 100644
--- a/src/table/src/kepler-table.ts
+++ b/src/table/src/kepler-table.ts
@@ -231,6 +231,8 @@ class KeplerTable<F extends Field = Field> {
sortOrder?: number[] | null;
pinnedColumns?: string[];
+ /** Attribute-table columns omitted from data export. */
+ hiddenColumns?: string[];
supportedFilterTypes?: string[] | null;
disableDataOperation?: boolean;
@@ -1013,6 +1015,30 @@ export function pinTableColumns(dataset: KeplerTable<Field>, column: string): Ke
return copyTableAndUpdate(dataset, {pinnedColumns});
}
+export function toggleHiddenTableColumn(
+ dataset: KeplerTable<Field>,
+ column: string
+): KeplerTable<Field> {
+ const field = dataset.fields.find(item => item.name === column);
+ if (!field) {
+ return dataset;
+ }
+
+ const hidden = dataset.hiddenColumns || [];
+ const hiddenColumns = hidden.includes(field.name)
+ ? hidden.filter(name => name !== field.name)
+ : hidden.concat(field.name);
+
+ return copyTableAndUpdate(dataset, {hiddenColumns});
+}
+
+export function showAllTableColumns(dataset: KeplerTable<Field>): KeplerTable<Field> {
+ if (!dataset.hiddenColumns?.length) {
+ return dataset;
+ }
+ return copyTableAndUpdate(dataset, {hiddenColumns: []});
+}
+
export function copyTable(original: KeplerTable<Field>): KeplerTable<Field> {
return Object.assign(Object.create(Object.getPrototypeOf(original)), original);
}
diff --git a/test/browser/components/modals/data-table-modal-test.js b/test/browser/components/modals/data-table-modal-test.js
index 9c0479c06..a2e73ad3d 100644
--- a/test/browser/components/modals/data-table-modal-test.js
+++ b/test/browser/components/modals/data-table-modal-test.js
@@ -225,23 +225,30 @@ test('Components -> DataTableModal.render: csv 1', t => {
const expectedColMeta = {
'gps_data.utc_timestamp': {
name: 'gps_data.utc_timestamp',
+ colIdx: 0,
type: 'timestamp',
format: 'YYYY-M-D H:m:s'
},
- 'gps_data.lat': {name: 'gps_data.lat', type: 'real'},
- 'gps_data.lng': {name: 'gps_data.lng', type: 'real'},
- 'gps_data.types': {name: 'gps_data.types', type: 'string'},
- epoch: {name: 'epoch', type: 'timestamp', format: 'X'},
- has_result: {name: 'has_result', type: 'boolean'},
- uid: {name: 'uid', type: 'integer'},
- time: {name: 'time', type: 'timestamp', format: 'YYYY-M-DTHH:mm:ss.SSSS'},
- begintrip_ts_utc: {name: 'begintrip_ts_utc', type: 'timestamp', format: 'YYYY-M-D HH:mm:ssZZ'},
+ 'gps_data.lat': {name: 'gps_data.lat', colIdx: 1, type: 'real'},
+ 'gps_data.lng': {name: 'gps_data.lng', colIdx: 2, type: 'real'},
+ 'gps_data.types': {name: 'gps_data.types', colIdx: 3, type: 'string'},
+ epoch: {name: 'epoch', colIdx: 4, type: 'timestamp', format: 'X'},
+ has_result: {name: 'has_result', colIdx: 5, type: 'boolean'},
+ uid: {name: 'uid', colIdx: 6, type: 'integer'},
+ time: {name: 'time', colIdx: 7, type: 'timestamp', format: 'YYYY-M-DTHH:mm:ss.SSSS'},
+ begintrip_ts_utc: {
+ name: 'begintrip_ts_utc',
+ colIdx: 8,
+ type: 'timestamp',
+ format: 'YYYY-M-D HH:mm:ssZZ'
+ },
begintrip_ts_local: {
name: 'begintrip_ts_local',
+ colIdx: 9,
type: 'timestamp',
format: 'YYYY-M-D HH:mm:ssZZ'
},
- date: {name: 'date', type: 'date', format: 'YYYY-M-D'}
+ date: {name: 'date', colIdx: 10, type: 'date', format: 'YYYY-M-D'}
};
t.deepEqual(props.columns, expectedColumns, 'DataTable should have the correct props.columns');
diff --git a/test/node/reducers/ui-state-test.js b/test/node/reducers/ui-state-test.js
index f8069fef8..fcced6624 100644
--- a/test/node/reducers/ui-state-test.js
+++ b/test/node/reducers/ui-state-test.js
@@ -260,13 +260,13 @@ test('#uiStateReducer -> SET_EXPORT_SELECTED_DATASET', t => {
});
test('#uiStateReducer -> SET_EXPORT_DATA_TYPE', t => {
- const newReducer = reducer(INITIAL_UI_STATE, setExportDataType(EXPORT_DATA_TYPE.JSON));
+ const newReducer = reducer(INITIAL_UI_STATE, setExportDataType(EXPORT_DATA_TYPE.GEOJSON));
const expectedState = {
...INITIAL_UI_STATE,
exportData: {
...INITIAL_UI_STATE.exportData,
- dataType: EXPORT_DATA_TYPE.JSON
+ dataType: EXPORT_DATA_TYPE.GEOJSON
}
};
diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.js
index 68f96db1b..f5d05b23c 100644
--- a/test/node/reducers/vis-state-test.js
+++ b/test/node/reducers/vis-state-test.js
@@ -6906,6 +6906,47 @@ test('#visStateReducer -> PIN_TABLE_COLUMN', t => {
t.end();
});
+test('#visStateReducer -> TOGGLE_TABLE_COLUMN_HIDDEN and SHOW_ALL_TABLE_COLUMNS', t => {
+ const initialState = CloneDeep(StateWFiles.visState);
+
+ const hiddenState = reducer(
+ initialState,
+ VisStateActions.toggleTableColumnHidden(testCsvDataId, 'gps_data.lat')
+ );
+ assertDatasetIsTable(t, hiddenState.datasets[testCsvDataId]);
+ t.deepEqual(
+ hiddenState.datasets[testCsvDataId].hiddenColumns,
+ ['gps_data.lat'],
+ 'should hide a table column through the reducer'
+ );
+
+ const shownState = reducer(
+ hiddenState,
+ VisStateActions.toggleTableColumnHidden(testCsvDataId, 'gps_data.lat')
+ );
+ t.deepEqual(shownState.datasets[testCsvDataId].hiddenColumns, [], 'should show a hidden column');
+
+ const allShownState = reducer(hiddenState, VisStateActions.showAllTableColumns(testCsvDataId));
+ t.deepEqual(allShownState.datasets[testCsvDataId].hiddenColumns, [], 'should show all columns');
+
+ const unknownColumnState = reducer(
+ initialState,
+ VisStateActions.toggleTableColumnHidden(testCsvDataId, 'missing')
+ );
+ t.equal(
+ unknownColumnState.datasets[testCsvDataId].hiddenColumns,
+ undefined,
+ 'should ignore an unknown column'
+ );
+ t.equal(
+ reducer(initialState, VisStateActions.toggleTableColumnHidden('missing', 'gps_data.lat')),
+ initialState,
+ 'should ignore an unknown dataset'
+ );
+
+ t.end();
+});
+
test('#visStateReducer -> LOAD_FILES', async t => {
drainTasksForTesting();
diff --git a/yarn.lock b/yarn.lock
index 027153df0..e1390f8b2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4354,8 +4354,11 @@ __metadata:
"@loaders.gl/zip": "npm:^4.4.1"
"@mapbox/geojson-normalize": "npm:0.0.1"
"@turf/helpers": "npm:^7.2.0"
+ "@types/sql.js": "npm:^1"
apache-arrow: "npm:>=15.0.0"
d3-dsv: "npm:^2.0.0"
+ parquet-wasm: "npm:0.6.1"
+ sql.js: "npm:1.13.0"
type-analyzer: "npm:0.4.0"
languageName: unknown
linkType: soft
@@ -10311,6 +10314,13 @@ __metadata:
languageName: node
linkType: hard
+"@types/emscripten@npm:*":
+ version: 1.41.6
+ resolution: "@types/emscripten@npm:1.41.6"
+ checksum: 10c0/b51a730b6cb2848ac5e3f0536707bf656b6887be21f20e96b01e74a59ed52319dd0942f49b1072b1ca6d38df19e2d633f540df54d6ba1f7216be14af7373a41f
+ languageName: node
+ linkType: hard
+
"@types/estree-jsx@npm:^1.0.0":
version: 1.0.5
resolution: "@types/estree-jsx@npm:1.0.5"
@@ -10922,6 +10932,16 @@ __metadata:
languageName: node
linkType: hard
+"@types/sql.js@npm:^1":
+ version: 1.4.11
+ resolution: "@types/sql.js@npm:1.4.11"
+ dependencies:
+ "@types/emscripten": "npm:*"
+ "@types/node": "npm:*"
+ checksum: 10c0/eefb0408235cb645879103414b8ef0f13c2e7f2eac69c2a6c2838b72ec0be49985d6c6711b95bc8d79c208c7c5db95a51576a6ce7bfb49dd9edcc17721ffb40f
+ languageName: node
+ linkType: hard
+
"@types/stack-utils@npm:^2.0.0":
version: 2.0.3
resolution: "@types/stack-utils@npm:2.0.3"
@@ -26343,7 +26363,7 @@ __metadata:
languageName: node
linkType: hard
-"parquet-wasm@npm:^0.6.1":
+"parquet-wasm@npm:0.6.1, parquet-wasm@npm:^0.6.1":
version: 0.6.1
resolution: "parquet-wasm@npm:0.6.1"
checksum: 10c0/dcade4b006ff86edca711dacf3c71997b4a86985384fc183119f73d5dae1d0c94d3e7fa5b48845845df5db97d3eb445d6c137f3459a5d9a66a3b9b3bc0474497
@@ -30543,6 +30563,13 @@ __metadata:
languageName: node
linkType: hard
+"sql.js@npm:1.13.0":
+ version: 1.13.0
+ resolution: "sql.js@npm:1.13.0"
+ checksum: 10c0/ecad62bf74082bf296a42db9bf001a79cd1b2fc954dbaddb97d689f7382c573ec827a692bd4490322fac47358d74c026258042466e258b34ea7c07b35d4a2e8e
+ languageName: node
+ linkType: hard
+
"ssf@npm:~0.11.2":
version: 0.11.2
resolution: "ssf@npm:0.11.2"