Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
feat(export-data): add GeoJSON, GeoParquet, Shapefile, GeoPackage, KML, and KMZ (#3811)
Igor Dykhtaebcf4e866ab7
diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.tsindex 6af6fed98..4c98a55df 100644--- a/src/actions/src/action-types.ts+++ b/src/actions/src/action-types.ts@@ -115,6 +115,8 @@ export const ActionTypes = {SET_MAP_INFO: `${ACTION_PREFIX}SET_MAP_INFO`,SORT_TABLE_COLUMN: `${ACTION_PREFIX}SORT_TABLE_COLUMN`,PIN_TABLE_COLUMN: `${ACTION_PREFIX}PIN_TABLE_COLUMN`,+ TOGGLE_TABLE_COLUMN_HIDDEN: `${ACTION_PREFIX}TOGGLE_TABLE_COLUMN_HIDDEN`,+ SHOW_ALL_TABLE_COLUMNS: `${ACTION_PREFIX}SHOW_ALL_TABLE_COLUMNS`,COPY_TABLE_COLUMN: `${ACTION_PREFIX}COPY_TABLE_COLUMN`,SET_COLUMN_DISPLAY_FORMAT: `${ACTION_PREFIX}SET_COLUMN_DISPLAY_FORMAT`,LOAD_COLUMN_STATS: `${ACTION_PREFIX}LOAD_COLUMN_STATS`,diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.tsindex c17cdfccb..ec68c1d64 100644--- a/src/actions/src/vis-state-actions.ts+++ b/src/actions/src/vis-state-actions.ts@@ -1251,6 +1251,49 @@ export function pinTableColumn(};}+export type ToggleTableColumnHiddenUpdaterAction = {+ dataId: string;+ column: string;+};+/**+ * Hide or show a dataset column in the attribute table and in data export.+ * @param dataId+ * @param column+ * @returns action+ * @public+ */+export function toggleTableColumnHidden(+ dataId: string,+ column: string+): Merge<+ ToggleTableColumnHiddenUpdaterAction,+ {type: typeof ActionTypes.TOGGLE_TABLE_COLUMN_HIDDEN}+> {+ return {+ type: ActionTypes.TOGGLE_TABLE_COLUMN_HIDDEN,+ dataId,+ column+ };+}++export type ShowAllTableColumnsUpdaterAction = {+ dataId: string;+};+/**+ * Show every attribute-table column that was hidden.+ * @param dataId+ * @returns action+ * @public+ */+export function showAllTableColumns(+ dataId: string+): Merge<ShowAllTableColumnsUpdaterAction, {type: typeof ActionTypes.SHOW_ALL_TABLE_COLUMNS}> {+ return {+ type: ActionTypes.SHOW_ALL_TABLE_COLUMNS,+ dataId+ };+}+export type CopyTableColumnUpdaterAction = {dataId: string;column: string;diff --git a/src/components/src/common/data-table/header-cell.tsx b/src/components/src/common/data-table/header-cell.tsxindex 196721a79..0b4e14c07 100644--- a/src/components/src/common/data-table/header-cell.tsx+++ b/src/components/src/common/data-table/header-cell.tsx@@ -200,6 +200,7 @@ function HeaderCellFactory(sortColumn,sortTableColumn,pinTableColumn,+ hideTableColumn,copyTableColumn,setColumnDisplayFormat,hasStats,@@ -254,6 +255,9 @@ function HeaderCellFactory(const onPin = useCallback(() => {if (!isGhost) pinTableColumn(column);}, [pinTableColumn, isGhost, column]);+ const onHide = useCallback(() => {+ if (!isGhost) hideTableColumn?.(column);+ }, [hideTableColumn, isGhost, column]);const onCopy = useCallback(() => {if (!isGhost) copyTableColumn(column);}, [copyTableColumn, isGhost, column]);@@ -375,6 +379,7 @@ function HeaderCellFactory(toggleMoreOptions={toggleMoreOptions}sortTableColumn={sortTableColumn ? mode => sortTableColumn(column, mode) : undefined}pinTableColumn={onPin}+ hideTableColumn={hideTableColumn ? onHide : undefined}copyTableColumn={onCopy}setDisplayFormat={setColumnDisplayFormat ? onSetDisplayFormat : undefined}renameTableColumn={renameTableColumn}diff --git a/src/components/src/common/data-table/index.tsx b/src/components/src/common/data-table/index.tsxindex 905c1c5cf..d7873d767 100644--- a/src/components/src/common/data-table/index.tsx+++ b/src/components/src/common/data-table/index.tsx@@ -259,14 +259,14 @@ interface GetRowCellProps {/** This is an accessor method used to generalize getting a cell from a data row*/-const defaultGetRowCell = (+export const defaultGetRowCell = ({dataContainer, columns, column, colMeta, rowIndex, sortOrder}: GetRowCellProps,formatter) => {const rowIdx = sortOrder && sortOrder.length ? get(sortOrder, rowIndex) : rowIndex;- const {type} = colMeta[column];+ const {type, colIdx} = colMeta[column];- const value = dataContainer?.valueAt(rowIdx, columns.indexOf(column));+ const value = dataContainer?.valueAt(rowIdx, colIdx ?? columns.indexOf(column));if (value === null || value === undefined || value === '') {return '';}@@ -454,6 +454,7 @@ export interface DataTableProps {sortColumn: SortColumn;sortTableColumn: (column: string, mode?: string) => void;pinTableColumn: (column: string) => void;+ hideTableColumn?: (column: string) => void;setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;copyTableColumn: (column: string) => void;renameTableColumn?: (column: string, newName: string) => void;diff --git a/src/components/src/common/data-table/option-dropdown.tsx b/src/components/src/common/data-table/option-dropdown.tsxindex 5d2bfa06f..b7d43216d 100644--- a/src/components/src/common/data-table/option-dropdown.tsx+++ b/src/components/src/common/data-table/option-dropdown.tsx@@ -23,7 +23,8 @@ import {Hash,Edit,Trash,- FilterFunnel+ FilterFunnel,+ EyeUnseen} from '../icons';const ListItem = ({value}) => (@@ -93,10 +94,10 @@ const StyledDeleteConfirm = styled.div`const StyledPopover = styled.div`width: 184px;- height: 160px;+ height: 200px;z-index: 101;.list-selector {- max-height: 160px;+ max-height: 200px;}.hover:after {content: '\\2713';@@ -158,6 +159,7 @@ export interface OptionDropdownProps {toggleMoreOptions: (column: string) => void;sortTableColumn?: (sort: string) => void;pinTableColumn: () => void;+ hideTableColumn?: () => void;copyTableColumn: () => void;setDisplayFormat?: (displayFormat: any) => void;sortMode?: string;@@ -177,6 +179,7 @@ const OptionDropdown = (props: OptionDropdownProps) => {toggleMoreOptions,sortTableColumn,pinTableColumn,+ hideTableColumn,copyTableColumn,setDisplayFormat,deleteTableColumn,@@ -206,6 +209,9 @@ const OptionDropdown = (props: OptionDropdownProps) => {case TABLE_OPTION.COPY:copyTableColumn();break;+ case TABLE_OPTION.HIDE:+ hideTableColumn?.();+ break;case TABLE_OPTION.FORMAT_COLUMN:setShowFormatter(true);return;@@ -228,6 +234,7 @@ const OptionDropdown = (props: OptionDropdownProps) => {column,sortTableColumn,pinTableColumn,+ hideTableColumn,copyTableColumn,toggleMoreOptions,onStartRename,@@ -244,7 +251,8 @@ const OptionDropdown = (props: OptionDropdownProps) => {Hash,Edit,Trash,- FilterFunnel+ FilterFunnel,+ EyeUnseen};const formatLabels = getFieldFormatLabels(colMeta[column].type);diff --git a/src/components/src/modal-container.spec.ts b/src/components/src/modal-container.spec.tsnew file mode 100644index 000000000..bd131c9cb--- /dev/null+++ b/src/components/src/modal-container.spec.ts@@ -0,0 +1,70 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {EXPORT_DATA_ID, EXPORT_MAP_ID} from '@kepler.gl/constants';++import {exportDataFromModal} from './modal-container';++function deferred() {+ let resolve!: () => void;+ let reject!: (error: Error) => void;+ const promise = new Promise<void>((resolvePromise, rejectPromise) => {+ resolve = resolvePromise;+ reject = rejectPromise;+ });+ return {promise, resolve, reject};+}++describe('exportDataFromModal', () => {+ test('closes the export modal after a successful export', async () => {+ const closeModal = jest.fn();+ await exportDataFromModal({+ exportData: () => Promise.resolve(),+ getCurrentModal: () => EXPORT_DATA_ID,+ closeModal,+ addNotification: jest.fn()+ });+ expect(closeModal).toHaveBeenCalledTimes(1);+ });++ test('does not close a newer modal after export completes', async () => {+ const exportTask = deferred();+ const closeModal = jest.fn();+ let currentModal = EXPORT_DATA_ID;+ const completion = exportDataFromModal({+ exportData: () => exportTask.promise,+ getCurrentModal: () => currentModal,+ closeModal,+ addNotification: jest.fn()+ });++ currentModal = EXPORT_MAP_ID;+ exportTask.resolve();+ await completion;++ expect(closeModal).not.toHaveBeenCalled();+ });++ test('shows an error notification and leaves the modal open when export fails', async () => {+ const exportTask = deferred();+ const closeModal = jest.fn();+ const addNotification = jest.fn();+ const completion = exportDataFromModal({+ exportData: () => exportTask.promise,+ getCurrentModal: () => EXPORT_DATA_ID,+ closeModal,+ addNotification+ });++ exportTask.reject(new Error('Unsupported geometry'));+ await completion;++ expect(closeModal).not.toHaveBeenCalled();+ expect(addNotification).toHaveBeenCalledWith({+ id: 'EXPORT_DATA_ERROR_ID',+ type: 'error',+ topic: 'global',+ message: 'Failed to export data: Unsupported geometry'+ });+ });+});diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsxindex 77242dd66..e03fb82ff 100644--- a/src/components/src/modal-container.tsx+++ b/src/components/src/modal-container.tsx@@ -125,6 +125,35 @@ export type ModalContainerProps = {onExportToCloudError?: OnErrorCallBack;};+type ExportDataModalActions = {+ exportData: () => Promise<void>;+ getCurrentModal: () => UiState['currentModal'];+ closeModal: () => void;+ addNotification: ModalContainerProps['uiStateActions']['addNotification'];+};++export async function exportDataFromModal({+ exportData: runExport,+ getCurrentModal,+ closeModal,+ addNotification+}: ExportDataModalActions): Promise<void> {+ try {+ await runExport();+ if (getCurrentModal() === EXPORT_DATA_ID) {+ closeModal();+ }+ } catch (error) {+ const detail = error instanceof Error ? error.message : String(error);+ addNotification({+ id: 'EXPORT_DATA_ERROR_ID',+ type: 'error',+ topic: 'global',+ message: `Failed to export data: ${detail}`+ });+ }+}+ModalContainerFactory.deps = [DeleteDatasetModalFactory,OverWriteMapModalFactory,@@ -304,8 +333,12 @@ export default function ModalContainerFactory(};_onExportData = () => {- exportData(this.props, this.props.uiState.exportData);- this._closeModal();+ return exportDataFromModal({+ exportData: () => exportData(this.props, this.props.uiState.exportData),+ getCurrentModal: () => this.props.uiState.currentModal,+ closeModal: this._closeModal,+ addNotification: this.props.uiStateActions.addNotification+ });};_onExportMap = () => {@@ -412,6 +445,8 @@ export default function ModalContainerFactory(showDatasetTable={visStateActions.showDatasetTable}sortTableColumn={visStateActions.sortTableColumn}pinTableColumn={visStateActions.pinTableColumn}+ toggleTableColumnHidden={visStateActions.toggleTableColumnHidden}+ showAllTableColumns={visStateActions.showAllTableColumns}copyTableColumn={visStateActions.copyTableColumn}renameTableColumn={visStateActions.renameTableColumn}deleteTableColumn={visStateActions.deleteTableColumn}diff --git a/src/components/src/modals/data-table-modal-column-edit.spec.tsx b/src/components/src/modals/data-table-modal-column-edit.spec.tsxindex c6889f40f..0de89921e 100644--- a/src/components/src/modals/data-table-modal-column-edit.spec.tsx+++ b/src/components/src/modals/data-table-modal-column-edit.spec.tsx@@ -7,6 +7,7 @@ import {ALL_FIELD_TYPES} from '@kepler.gl/constants';import {KeplerTable} from '@kepler.gl/table';import {renderWithTheme} from 'test/helpers/component-jest-utils';+import {defaultGetRowCell} from '../common/data-table';import DataTableModalFactory from './data-table-modal';jest.mock('../common/data-table/cell-size', () => ({@@ -30,7 +31,7 @@ function StubDataTable(props: {const DataTableModal = DataTableModalFactory(StubDataTable as never, (() => null) as never);-function makeDataset(type: string) {+function makeDataset(type: string, hiddenColumns: string[] = []) {const table = new KeplerTable({info: {id: 'cities', label: 'cities', type},color: [0, 0, 0]@@ -38,14 +39,16 @@ function makeDataset(type: string) {table.updateSchema({fields: [{name: 'region', type: ALL_FIELD_TYPES.string},+ {name: 'secret', type: ALL_FIELD_TYPES.string},{name: 'pop', type: ALL_FIELD_TYPES.integer}],- rows: [['west', 1]]+ rows: [['west', 'hidden-value', 1]]});+ table.hiddenColumns = hiddenColumns;return table;}-function renderModal(type: string) {+function renderModal(type: string, hiddenColumns: string[] = []) {const createFilterFromColumn = jest.fn();const renameTableColumn = jest.fn();const deleteTableColumn = jest.fn();@@ -54,7 +57,7 @@ function renderModal(type: string) {const view = renderWithTheme(<DataTableModaldataId="cities"- datasets={{cities: makeDataset(type)}}+ datasets={{cities: makeDataset(type, hiddenColumns)}}showTab={false}sortTableColumn={jest.fn()}pinTableColumn={jest.fn()}@@ -91,4 +94,27 @@ describe('data table column edits', () => {expect(tableProps.current?.renameTableColumn).toBeUndefined();expect(tableProps.current?.deleteTableColumn).toBeUndefined();});++ test('keeps original field indexes when a middle column is hidden', () => {+ renderModal('local', ['secret']);++ const props = tableProps.current as {+ columns: string[];+ colMeta: Record<string, {colIdx: number; type: string}>;+ dataContainer: {valueAt: (row: number, column: number) => unknown};+ };+ expect(props.columns).toEqual(['region', 'pop']);+ expect(props.colMeta.pop.colIdx).toBe(2);+ expect(+ defaultGetRowCell(+ {+ ...props,+ column: 'pop',+ rowIndex: 0,+ sortOrder: null+ } as never,+ null+ )+ ).toBe('1');+ });});diff --git a/src/components/src/modals/data-table-modal.tsx b/src/components/src/modals/data-table-modal.tsxindex 6fcc5092e..9ae8fe29e 100644--- a/src/components/src/modals/data-table-modal.tsx+++ b/src/components/src/modals/data-table-modal.tsx@@ -70,8 +70,19 @@ export const DatasetModalTab: IStyledComponent<}`;+const HiddenColumnsBar = styled.button`+ margin-right: 12px;+ border: 0;+ background: transparent;+ color: ${props => props.theme.subtextColor};+ cursor: pointer;+ font-size: 11px;+ padding: 0;+`;+const StyledConfigureButton = styled.div`display: flex;+ align-items: center;justify-content: flex-end;position: absolute;top: 24px;@@ -129,6 +140,8 @@ interface DataTableModalProps {dataId?: string;sortTableColumn: (id: string, column: string, mode?: string) => void;pinTableColumn: (id: string, column: string) => void;+ toggleTableColumnHidden?: (id: string, column: string) => void;+ showAllTableColumns?: (id: string) => void;copyTableColumn: (id: string, column: string) => void;renameTableColumn?: (id: string, column: string, newName: string) => void;deleteTableColumn?: (id: string, column: string) => void;@@ -157,16 +170,17 @@ function DataTableModalFactory(dataId = '',sortTableColumn,pinTableColumn,+ toggleTableColumnHidden,+ showAllTableColumns,copyTableColumn: copyTableColumnProp,renameTableColumn,deleteTableColumn,createFilterFromColumn,datasets,showDatasetTable,+ uiStateActions,showTab = true,setColumnDisplayFormat: setColumnDisplayFormatProp,- uiStateActions,- uiState,loadColumnStats}) => {const [showConfig, setShowConfig] = useState(false);@@ -179,15 +193,27 @@ function DataTableModalFactory([datasets, dataId]);- const columns = useMemo(() => fields?.map(f => f.name) || [], [fields]);+ const hiddenColumns = useMemo(+ () => datasets?.[dataId]?.hiddenColumns ?? [],+ [datasets, dataId]+ );+ const columns = useMemo(() => {+ const hidden = new Set(hiddenColumns);+ return (fields?.map(field => field.name) || []).filter(name => !hidden.has(name));+ }, [fields, hiddenColumns]);const colMeta = useMemo(() =>fields?.reduce(- (acc, {name, displayName, type, filterProps, format, displayFormat, isLoadingStats}) => ({+ (+ acc,+ {name, displayName, type, filterProps, format, displayFormat, isLoadingStats},+ colIdx+ ) => ({...acc,[name]: {name: displayName || name,+ colIdx,type,...(format ? {format} : {}),...(displayFormat ? {displayFormat} : {}),@@ -267,6 +293,17 @@ function DataTableModalFactory([pinTableColumn, dataId]);+ const handleHideTableColumn = useCallback(+ (column: string) => {+ toggleTableColumnHidden?.(dataId, column);+ },+ [toggleTableColumnHidden, dataId]+ );++ const handleShowAllColumns = useCallback(() => {+ showAllTableColumns?.(dataId);+ }, [showAllTableColumns, dataId]);+const handleSortTableColumn = useCallback((column: string, mode?: string) => {sortTableColumn(dataId, column, mode);@@ -332,6 +369,11 @@ function DataTableModalFactory(/>) : null}<StyledConfigureButton className="display-config-button">+ {hiddenColumns.length ? (+ <HiddenColumnsBar type="button" onClick={handleShowAllColumns}>+ {`Show ${hiddenColumns.length} hidden`}+ </HiddenColumnsBar>+ ) : null}<Gear onClick={onOpenConfig} /><Portaled right={240} top={20} isOpened={showConfig} onClose={onCloseConfig}><DataTableConfig@@ -350,11 +392,14 @@ function DataTableModalFactory(colMeta={colMeta}cellSizeCache={cellSizeCache}dataContainer={activeDataset.dataContainer}- pinnedColumns={activeDataset.pinnedColumns}+ pinnedColumns={(activeDataset.pinnedColumns || []).filter(+ column => !hiddenColumns.includes(column)+ )}sortOrder={activeDataset.sortOrder}sortColumn={activeDataset.sortColumn || DEFAULT_SORT_COLUMN}copyTableColumn={handleCopyTableColumn}pinTableColumn={handlePinTableColumn}+ hideTableColumn={toggleTableColumnHidden ? handleHideTableColumn : undefined}sortTableColumn={handleSortTableColumn}setColumnDisplayFormat={handleSetColumnDisplayFormat}renameTableColumn={diff --git a/src/components/src/modals/export-data-modal.spec.tsx b/src/components/src/modals/export-data-modal.spec.tsxnew file mode 100644index 000000000..8add9aeff--- /dev/null+++ b/src/components/src/modals/export-data-modal.spec.tsx@@ -0,0 +1,50 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import React from 'react';+import {fireEvent} from '@testing-library/react';++import {EXPORT_DATA_TYPE, EXPORT_DATA_TYPE_OPTIONS} from '@kepler.gl/constants';+import {Datasets} from '@kepler.gl/table';+import {renderWithTheme} from 'test/helpers/component-jest-utils';++import ExportDataModalFactory from './export-data-modal';++const ExportDataModal = ExportDataModalFactory();++const datasets = {+ trips: {+ label: 'Trips',+ dataContainer: {numRows: () => 2},+ filteredIdxCPU: [0]+ }+};++describe('ExportDataModal', () => {+ test('data type is a dropdown that defaults to csv', () => {+ const onChangeExportDataType = jest.fn();+ const {container} = renderWithTheme(+ <ExportDataModal+ datasets={datasets as unknown as Datasets}+ selectedDataset="trips"+ dataType={EXPORT_DATA_TYPE.CSV}+ filtered={false}+ applyCPUFilter={jest.fn()}+ onChangeExportSelectedDataset={jest.fn()}+ onChangeExportDataType={onChangeExportDataType}+ onChangeExportFiltered={jest.fn()}+ />+ );++ const dataType = container.querySelector('select.data-type-select') as HTMLSelectElement;+ expect(dataType).not.toBeNull();+ expect(dataType.value).toBe(EXPORT_DATA_TYPE.CSV);+ expect(Array.from(dataType.options).map(option => option.text)).toEqual(+ EXPORT_DATA_TYPE_OPTIONS.map(option => option.label)+ );+ expect(dataType.parentElement?.children).toHaveLength(1);++ fireEvent.change(dataType, {target: {value: EXPORT_DATA_TYPE.GEOJSON}});+ expect(onChangeExportDataType).toHaveBeenCalledWith(EXPORT_DATA_TYPE.GEOJSON);+ });+});diff --git a/src/components/src/modals/export-data-modal.tsx b/src/components/src/modals/export-data-modal.tsxindex a1a7c01bf..b54532906 100644--- a/src/components/src/modals/export-data-modal.tsx+++ b/src/components/src/modals/export-data-modal.tsx@@ -9,12 +9,10 @@ import {DatasetType, EXPORT_DATA_TYPE_OPTIONS} from '@kepler.gl/constants';import {FormattedMessage} from '@kepler.gl/localization';import {Datasets} from '@kepler.gl/table';-import {FileType} from '../common/icons';import {StyledExportSection,StyledFilteredOption,StyledModalContent,- StyledType,CheckMark} from '../common/styled-components';import {StyledWarning} from './export-map-modal/components';@@ -64,7 +62,7 @@ export interface ExportDataModalProps {onChangeExportDataType: (type: string) => void;onChangeExportFiltered: (isFiltered: boolean) => void;intl: IntlShape;- supportedDataTypes: {+ supportedDataTypes?: {id: string;label: string;available: boolean;@@ -95,6 +93,10 @@ const ExportDataModalFactory = () => {onChangeExportSelectedDataset(value);};+ const onSelectDataType: React.ChangeEventHandler<HTMLSelectElement> = ({target: {value}}) => {+ onChangeExportDataType(value);+ };+const exportAllDatasets = selectedDataset ? !datasets[selectedDataset] : true;const showTiledDatasetWarning = Object.keys(datasets).some(datasetId => {return (@@ -141,16 +143,13 @@ const ExportDataModalFactory = () => {</div></div><div className="selection">- {supportedDataTypes.map(op => (- <StyledType- key={op.id}- selected={dataType === op.id}- onClick={() => op.available && onChangeExportDataType(op.id)}- >- <FileType ext={op.label} height="80px" fontSize="11px" />- {dataType === op.id && <CheckMark />}- </StyledType>- ))}+ <select className="data-type-select" value={dataType} onChange={onSelectDataType}>+ {supportedDataTypes.map(op => (+ <option key={op.id} value={op.id} disabled={!op.available}>+ {op.label}+ </option>+ ))}+ </select></div></CompactExportSection><CompactExportSection>diff --git a/src/constants/src/default-settings.ts b/src/constants/src/default-settings.tsindex 9dd350e81..217604142 100644--- a/src/constants/src/default-settings.ts+++ b/src/constants/src/default-settings.ts@@ -539,6 +539,7 @@ export const TABLE_OPTION = keyMirror({UNPIN: null,COPY: null,FORMAT_COLUMN: null,+ HIDE: null,RENAME: null,DELETE: null,CREATE_FILTER: null@@ -553,6 +554,7 @@ export type TableOption = {| 'UNPIN'| 'COPY'| 'FORMAT_COLUMN'+ | 'HIDE'| 'RENAME'| 'DELETE'| 'CREATE_FILTER';@@ -592,6 +594,12 @@ export const TABLE_OPTION_LIST: TableOption[] = [condition: props => props.isPinned},{value: TABLE_OPTION.COPY, display: 'Copy Column', icon: 'Clipboard'},+ {+ value: TABLE_OPTION.HIDE,+ display: 'Hide Column',+ icon: 'EyeUnseen',+ condition: props => props.hideTableColumn+ },{value: TABLE_OPTION.FORMAT_COLUMN,display: 'Format Column',@@ -1194,39 +1202,51 @@ export const EXPORT_IMG_RESOLUTION_OPTIONS: ReadonlyArray<ImageResolutionOption>];export const EXPORT_DATA_TYPE = keyMirror({- CSV: null- // SHAPEFILE: null,- // JSON: null,- // GEOJSON: null,- // TOPOJSON: null+ CSV: null,+ GEOJSON: null,+ GEOPARQUET: null,+ SHAPEFILE: null,+ GEOPACKAGE: null,+ KML: null,+ KMZ: null});export const EXPORT_DATA_TYPE_OPTIONS = [{id: EXPORT_DATA_TYPE.CSV,- label: EXPORT_DATA_TYPE.CSV.toLowerCase(),+ label: 'csv',+ available: true+ },+ {+ id: EXPORT_DATA_TYPE.GEOJSON,+ label: 'geojson',+ available: true+ },+ {+ id: EXPORT_DATA_TYPE.GEOPARQUET,+ label: 'parquet',+ available: true+ },+ {+ id: EXPORT_DATA_TYPE.SHAPEFILE,+ label: 'shp',+ available: true+ },+ {+ id: EXPORT_DATA_TYPE.GEOPACKAGE,+ label: 'gpkg',+ available: true+ },+ {+ id: EXPORT_DATA_TYPE.KML,+ label: 'kml',+ available: true+ },+ {+ id: EXPORT_DATA_TYPE.KMZ,+ label: 'kmz',available: true}- // {- // id: EXPORT_DATA_TYPE.SHAPEFILE,- // label: 'shapefile',- // available: false- // },- // {- // id: EXPORT_DATA_TYPE.JSON,- // label: 'json',- // available: false- // },- // {- // id: EXPORT_DATA_TYPE.GEOJSON,- // label: 'geojson',- // available: false- // },- // {- // id: EXPORT_DATA_TYPE.TOPOJSON,- // label: 'topojson',- // available: false- // }];// Export map typesdiff --git a/src/processors/package.json b/src/processors/package.jsonindex 3d97730d2..d659796f6 100644--- a/src/processors/package.json+++ b/src/processors/package.json@@ -53,6 +53,8 @@"@turf/helpers": "^7.2.0","apache-arrow": ">=15.0.0","d3-dsv": "^2.0.0",+ "parquet-wasm": "0.6.1",+ "sql.js": "1.13.0","type-analyzer": "0.4.0"},"nyc": {@@ -81,5 +83,8 @@"require": "./dist/cjs/index.js","default": "./dist/cjs/index.js"}+ },+ "devDependencies": {+ "@types/sql.js": "^1"}}diff --git a/src/processors/src/export/dataset-to-geojson.ts b/src/processors/src/export/dataset-to-geojson.tsnew file mode 100644index 000000000..55d6fbbe0--- /dev/null+++ b/src/processors/src/export/dataset-to-geojson.ts@@ -0,0 +1,153 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {h3IsValid, idToPolygonGeo} from '@kepler.gl/common-utils';+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';+import {parseGeometry} from '@kepler.gl/table';+import {Field, FieldPair} from '@kepler.gl/types';+import {DataContainerInterface, parseFieldValue} from '@kepler.gl/utils';+import {convertWKBToGeometry, convertWKTToGeometry} from '@loaders.gl/gis';+import type {Feature, FeatureCollection, Geometry, Position} from 'geojson';++type GeometryPlan =+ | {kind: 'geo'; fieldIndex: number}+ | {kind: 'latlng'; pair: FieldPair}+ | {kind: 'h3'; fieldIndex: number}+ | {kind: 'none'};++function geometryPlan(fields: Field[], fieldPairs: FieldPair[] = []): GeometryPlan {+ const geoIndex = fields.findIndex(+ field =>+ field.type === ALL_FIELD_TYPES.geojson ||+ field.type === ALL_FIELD_TYPES.geoarrow ||+ field.type === ALL_FIELD_TYPES.point+ );+ if (geoIndex >= 0) {+ return {kind: 'geo', fieldIndex: geoIndex};+ }+ if (fieldPairs[0]) {+ return {kind: 'latlng', pair: fieldPairs[0]};+ }+ const h3Index = fields.findIndex(field => field.type === ALL_FIELD_TYPES.h3);+ if (h3Index >= 0) {+ return {kind: 'h3', fieldIndex: h3Index};+ }+ return {kind: 'none'};+}++function geometryFromBinary(raw: ArrayBuffer | ArrayBufferView): Geometry | null {+ try {+ const bytes =+ raw instanceof ArrayBuffer+ ? raw+ : raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength);+ return convertWKBToGeometry(bytes) as Geometry;+ } catch {+ return null;+ }+}++function geometryFromRaw(raw: unknown, fieldType: string): Geometry | null {+ if (raw == null) {+ return null;+ }+ if (raw instanceof ArrayBuffer || ArrayBuffer.isView(raw)) {+ return geometryFromBinary(raw);+ }+ const parsed = parseGeometry(raw);+ if (parsed?.geometry) {+ return parsed.geometry;+ }+ if (fieldType === ALL_FIELD_TYPES.geoarrow && typeof raw === 'string') {+ const text = raw.trim();+ if (/^(POINT|LINESTRING|POLYGON|MULTI|GEOMETRYCOLLECTION)\b/i.test(text)) {+ try {+ return convertWKTToGeometry(text) as Geometry;+ } catch {+ return null;+ }+ }+ }+ return null;+}++function attributeValue(value: unknown, field: Field): string | number | boolean | null {+ if (value == null) {+ return null;+ }+ if (typeof value === 'number') {+ return Number.isFinite(value) ? value : null;+ }+ if (typeof value === 'boolean' || typeof value === 'string') {+ return value;+ }+ if (typeof value === 'bigint') {+ return value.toString();+ }+ const formatted = parseFieldValue(value, field.type, field);+ return formatted === '' ? null : formatted;+}++function pointGeometry(+ data: DataContainerInterface,+ rowIndex: number,+ pair: FieldPair+): Geometry | null {+ const lat = Number(data.valueAt(rowIndex, pair.pair.lat.fieldIdx));+ const lng = Number(data.valueAt(rowIndex, pair.pair.lng.fieldIdx));+ if (!Number.isFinite(lat) || !Number.isFinite(lng)) {+ return null;+ }+ const altitudeIndex = pair.pair.altitude?.fieldIdx;+ const altitude =+ altitudeIndex === undefined ? undefined : Number(data.valueAt(rowIndex, altitudeIndex));+ const coordinates: Position =+ altitude !== undefined && Number.isFinite(altitude) ? [lng, lat, altitude] : [lng, lat];+ return {type: 'Point', coordinates};+}++/**+ * Turn dataset rows into a FeatureCollection.+ * Columns listed in `hiddenColumns` are left out of feature properties.+ * A geometry column is still used to build the feature geometry.+ */+export function datasetToFeatureCollection(+ data: DataContainerInterface,+ fields: Field[],+ options: {hiddenColumns?: string[]; fieldPairs?: FieldPair[]} = {}+): FeatureCollection {+ const hidden = new Set(options.hiddenColumns ?? []);+ const plan = geometryPlan(fields, options.fieldPairs);+ const geometryFieldIndex = plan.kind === 'geo' || plan.kind === 'h3' ? plan.fieldIndex : -1;+ const features: Feature[] = [];++ for (let rowIndex = 0; rowIndex < data.numRows(); rowIndex++) {+ const properties: Record<string, string | number | boolean | null> = {};+ fields.forEach((field, fieldIndex) => {+ if (hidden.has(field.name) || fieldIndex === geometryFieldIndex) {+ return;+ }+ properties[field.name] = attributeValue(data.valueAt(rowIndex, fieldIndex), field);+ });++ let geometry: Geometry | null = null;+ if (plan.kind === 'geo') {+ geometry = geometryFromRaw(+ data.valueAt(rowIndex, plan.fieldIndex),+ fields[plan.fieldIndex].type+ );+ } else if (plan.kind === 'latlng') {+ geometry = pointGeometry(data, rowIndex, plan.pair);+ } else if (plan.kind === 'h3') {+ const index = String(data.valueAt(rowIndex, plan.fieldIndex) ?? '');+ geometry =+ (h3IsValid(index)+ ? (idToPolygonGeo({id: index}, {isClosed: true})?.geometry as Geometry | undefined)+ : null) ?? null;+ }++ features.push({type: 'Feature', geometry: geometry as Geometry, properties});+ }++ return {type: 'FeatureCollection', features};+}diff --git a/src/processors/src/export/geopackage-writer.ts b/src/processors/src/export/geopackage-writer.tsnew file mode 100644index 000000000..1f91325a0--- /dev/null+++ b/src/processors/src/export/geopackage-writer.ts@@ -0,0 +1,392 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project+// Adapted from GeoLibre (https://github.com/opengeos/GeoLibre), MIT License.++import {convertGeometryToWKB} from '@loaders.gl/gis';+import type {FeatureCollection, Geometry, Position} from 'geojson';+import type {Database, SqlJsStatic} from 'sql.js';++import {loadSqlJs} from './sql-js';++/**+ * Write a GeoJSON FeatureCollection to an OGC GeoPackage (a single SQLite+ * database) entirely in the browser via sql.js.+ *+ * DuckDB-WASM cannot write the SQLite-backed GeoPackage format (its virtual+ * filesystem lacks the random-access seek/write the driver needs) and Pyodide's+ * bundled GDAL has no working GeoPackage write driver, so the file is assembled+ * directly here. Output carries one feature layer in WGS84 (EPSG:4326) with the+ * required `gpkg_*` metadata tables plus `gpkg_ogr_contents`, so it reopens in+ * GeoLibre, QGIS, and ArcGIS.+ */++// SQLite "application_id" for a GeoPackage: ASCII "GPKG" as a big-endian int32.+const GPKG_APPLICATION_ID = 0x47504b47;+// SQLite "user_version" for GeoPackage 1.3 (10300), per the spec's versioning.+const GPKG_USER_VERSION = 10300;+const SRS_ID = 4326;+const GEOMETRY_COLUMN = 'geom';+const FID_COLUMN = 'fid';+const GPKG_METADATA_TABLES = new Set([+ 'gpkg_spatial_ref_sys',+ 'gpkg_contents',+ 'gpkg_geometry_columns',+ 'gpkg_ogr_contents'+]);++const WGS84_DEFINITION =+ 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,' ++ 'AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],' ++ 'PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],' ++ 'UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],' ++ 'AUTHORITY["EPSG","4326"]]';++type ColumnType = 'INTEGER' | 'REAL' | 'TEXT';++interface ColumnSpec {+ name: string;+ type: ColumnType;+}++interface BoundingBox {+ minX: number;+ minY: number;+ maxX: number;+ maxY: number;+}++function emptyBoundingBox(): BoundingBox {+ return {minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity};+}++function extendBoundingBox(box: BoundingBox, geometry: Geometry): void {+ const visit = (position: Position) => {+ const x = Number(position[0]);+ const y = Number(position[1]);+ if (Number.isFinite(x)) {+ box.minX = Math.min(box.minX, x);+ box.maxX = Math.max(box.maxX, x);+ }+ if (Number.isFinite(y)) {+ box.minY = Math.min(box.minY, y);+ box.maxY = Math.max(box.maxY, y);+ }+ };+ const walk = (geom: Geometry) => {+ switch (geom.type) {+ case 'Point':+ visit(geom.coordinates);+ break;+ case 'LineString':+ case 'MultiPoint':+ geom.coordinates.forEach(visit);+ break;+ case 'Polygon':+ case 'MultiLineString':+ geom.coordinates.forEach(part => part.forEach(visit));+ break;+ case 'MultiPolygon':+ geom.coordinates.forEach(polygon => polygon.forEach(ring => ring.forEach(visit)));+ break;+ case 'GeometryCollection':+ geom.geometries.forEach(walk);+ break;+ default:+ break;+ }+ };+ walk(geometry);+}++function quoteIdentifier(value: string): string {+ return `"${value.replace(/"/g, '""')}"`;+}++/** Strip characters a SQLite identifier should not carry and avoid collisions+ * with the reserved fid/geom columns. */+function sanitizeColumnName(name: string, taken: Set<string>): string {+ let cleaned = name.replace(/[^\p{L}\p{N}_]+/gu, '_').replace(/^_+/, '');+ if (!cleaned) cleaned = 'field';+ if (cleaned.toLowerCase() === FID_COLUMN || cleaned.toLowerCase() === GEOMETRY_COLUMN) {+ cleaned = `${cleaned}_`;+ }+ let candidate = cleaned;+ let index = 1;+ while (taken.has(candidate.toLowerCase())) {+ candidate = `${cleaned}_${index}`;+ index += 1;+ }+ taken.add(candidate.toLowerCase());+ return candidate;+}++function sanitizeTableName(name: string): string {+ const candidate = sanitizeColumnName(name, new Set(GPKG_METADATA_TABLES));+ return /^sqlite_/i.test(candidate) ? `layer_${candidate}` : candidate;+}++/** Infer a SQLite column type from a property's values across all features. */+function inferColumnType(values: unknown[]): ColumnType {+ let sawNumber = false;+ let allInteger = true;+ for (const value of values) {+ if (value == null) continue;+ if (typeof value === 'number' && Number.isFinite(value)) {+ sawNumber = true;+ if (!Number.isInteger(value)) allInteger = false;+ } else if (typeof value === 'boolean') {+ sawNumber = true; // store booleans as 0/1 integers+ } else {+ return 'TEXT';+ }+ }+ if (!sawNumber) return 'TEXT';+ return allInteger ? 'INTEGER' : 'REAL';+}++/** Coerce a property value to what a SQLite bind expects for `type`. */+function coerceValue(value: unknown, type: ColumnType): number | string | null {+ if (value == null) return null;+ if (type === 'TEXT') {+ return typeof value === 'object' ? JSON.stringify(value) : String(value);+ }+ if (typeof value === 'boolean') return value ? 1 : 0;+ if (typeof value === 'number' && Number.isFinite(value)) return value;+ return null;+}++/**+ * Build the GeoPackage geometry blob: the "GP" header (version, flags, srs_id)+ * followed by the standard WKB. Flags `0x01` mean little-endian header fields+ * and no envelope. Returns null for a null/empty geometry so the row's geom+ * column is SQL NULL.+ */+function geoPackageBlob(geometry: Geometry | null): Uint8Array | null {+ if (!geometry) return null;+ const wkb = new Uint8Array(convertGeometryToWKB(geometry));+ const header = new Uint8Array(8);+ const view = new DataView(header.buffer);+ header[0] = 0x47; // 'G'+ header[1] = 0x50; // 'P'+ header[2] = 0x00; // version 0+ header[3] = 0x01; // flags: little-endian, no envelope, standard, non-empty+ view.setInt32(4, SRS_ID, true);+ const blob = new Uint8Array(header.length + wkb.length);+ blob.set(header, 0);+ blob.set(wkb, header.length);+ return blob;+}++function geometryTypeName(types: Set<string>): string {+ if (types.size === 1) {+ return [...types][0].toUpperCase();+ }+ return 'GEOMETRY';+}++function createMetadataTables(db: Database): void {+ db.run(`PRAGMA application_id = ${GPKG_APPLICATION_ID};`);+ db.run(`PRAGMA user_version = ${GPKG_USER_VERSION};`);+ db.run(+ `CREATE TABLE gpkg_spatial_ref_sys (+ srs_name TEXT NOT NULL,+ srs_id INTEGER NOT NULL PRIMARY KEY,+ organization TEXT NOT NULL,+ organization_coordsys_id INTEGER NOT NULL,+ definition TEXT NOT NULL,+ description TEXT+ );`+ );+ const srs = db.prepare(`INSERT INTO gpkg_spatial_ref_sys VALUES (?, ?, ?, ?, ?, ?);`);+ try {+ srs.run(['Undefined cartesian SRS', -1, 'NONE', -1, 'undefined', null]);+ srs.run(['Undefined geographic SRS', 0, 'NONE', 0, 'undefined', null]);+ srs.run(['WGS 84 geodetic', SRS_ID, 'EPSG', SRS_ID, WGS84_DEFINITION, null]);+ } finally {+ srs.free();+ }++ db.run(+ `CREATE TABLE gpkg_contents (+ table_name TEXT NOT NULL PRIMARY KEY,+ data_type TEXT NOT NULL,+ identifier TEXT UNIQUE,+ description TEXT DEFAULT '',+ last_change DATETIME NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),+ min_x DOUBLE,+ min_y DOUBLE,+ max_x DOUBLE,+ max_y DOUBLE,+ srs_id INTEGER,+ CONSTRAINT fk_gc_r_srs_id FOREIGN KEY (srs_id)+ REFERENCES gpkg_spatial_ref_sys(srs_id)+ );`+ );+ db.run(+ `CREATE TABLE gpkg_geometry_columns (+ table_name TEXT NOT NULL,+ column_name TEXT NOT NULL,+ geometry_type_name TEXT NOT NULL,+ srs_id INTEGER NOT NULL,+ z TINYINT NOT NULL,+ m TINYINT NOT NULL,+ CONSTRAINT pk_geom_cols PRIMARY KEY (table_name, column_name),+ CONSTRAINT uk_gc_table_name UNIQUE (table_name),+ CONSTRAINT fk_gc_tn FOREIGN KEY (table_name)+ REFERENCES gpkg_contents(table_name),+ CONSTRAINT fk_gc_srs FOREIGN KEY (srs_id)+ REFERENCES gpkg_spatial_ref_sys(srs_id)+ );`+ );+ db.run(+ `CREATE TABLE gpkg_ogr_contents (+ table_name TEXT NOT NULL PRIMARY KEY,+ feature_count INTEGER DEFAULT NULL+ );`+ );+}++/**+ * Synchronous core of {@link writeGeoPackage}, separated so it can be unit-tested+ * with an already-initialised sql.js factory (the async wrapper resolves the+ * factory through a Vite `?url` import that only works in the browser).+ *+ * Args:+ * SQL: An initialised sql.js factory.+ * geojson: Features to write (assumed WGS84).+ * layerName: Name of the single feature table / layer in the GeoPackage.+ *+ * Returns:+ * The GeoPackage file as bytes.+ */+export function writeGeoPackageSync(+ SQL: SqlJsStatic,+ geojson: FeatureCollection,+ layerName: string+): Uint8Array {+ const features = geojson.features ?? [];+ if (features.length === 0) {+ throw new Error('The layer has no features to export.');+ }++ const tableName = sanitizeTableName(layerName);++ // Collect the union of property keys (features may carry different keys).+ const propertyKeys: string[] = [];+ const seenKeys = new Set<string>();+ for (const feature of features) {+ for (const key of Object.keys(feature.properties ?? {})) {+ if (!seenKeys.has(key)) {+ seenKeys.add(key);+ propertyKeys.push(key);+ }+ }+ }++ const takenColumns = new Set<string>([FID_COLUMN, GEOMETRY_COLUMN]);+ const columns: (ColumnSpec & {key: string})[] = propertyKeys.map(key => ({+ key,+ name: sanitizeColumnName(key, takenColumns),+ type: inferColumnType(features.map(f => f.properties?.[key]))+ }));++ const box: BoundingBox = emptyBoundingBox();+ const geometryTypes = new Set<string>();+ for (const feature of features) {+ if (feature.geometry) {+ geometryTypes.add(feature.geometry.type);+ extendBoundingBox(box, feature.geometry);+ }+ }++ const db = new SQL.Database();+ try {+ createMetadataTables(db);++ const columnDefs = columns+ .map(column => `${quoteIdentifier(column.name)} ${column.type}`)+ .join(', ');+ db.run(+ `CREATE TABLE ${quoteIdentifier(tableName)} (+ ${quoteIdentifier(FID_COLUMN)} INTEGER PRIMARY KEY AUTOINCREMENT,+ ${quoteIdentifier(GEOMETRY_COLUMN)} BLOB${columnDefs ? `, ${columnDefs}` : ''}+ );`+ );++ const insertColumns = [GEOMETRY_COLUMN, ...columns.map(column => column.name)];+ const placeholders = insertColumns.map(() => '?').join(', ');+ const insert = db.prepare(+ `INSERT INTO ${quoteIdentifier(tableName)} (${insertColumns+ .map(quoteIdentifier)+ .join(', ')}) VALUES (${placeholders});`+ );+ try {+ db.run('BEGIN;');+ for (const feature of features) {+ const row: (number | string | Uint8Array | null)[] = [geoPackageBlob(feature.geometry)];+ for (const column of columns) {+ row.push(coerceValue(feature.properties?.[column.key], column.type));+ }+ insert.run(row);+ }+ db.run('COMMIT;');+ } finally {+ insert.free();+ }++ const finiteBox = Number.isFinite(box.minX) && Number.isFinite(box.minY);+ const contents = db.prepare(+ `INSERT INTO gpkg_contents+ (table_name, data_type, identifier, description, min_x, min_y, max_x, max_y, srs_id)+ VALUES (?, 'features', ?, '', ?, ?, ?, ?, ?);`+ );+ try {+ contents.run([+ tableName,+ tableName,+ finiteBox ? box.minX : null,+ finiteBox ? box.minY : null,+ finiteBox ? box.maxX : null,+ finiteBox ? box.maxY : null,+ SRS_ID+ ]);+ } finally {+ contents.free();+ }++ db.run(`INSERT INTO gpkg_geometry_columns VALUES (?, ?, ?, ?, 0, 0);`, [+ tableName,+ GEOMETRY_COLUMN,+ geometryTypeName(geometryTypes),+ SRS_ID+ ]);+ db.run(`INSERT INTO gpkg_ogr_contents (table_name, feature_count) VALUES (?, ?);`, [+ tableName,+ features.length+ ]);++ // sql.js always exports an ArrayBuffer-backed Uint8Array.+ return db.export();+ } finally {+ db.close();+ }+}++/**+ * Write a GeoJSON FeatureCollection to a GeoPackage (loads sql.js, then defers+ * to {@link writeGeoPackageSync}).+ *+ * Args:+ * geojson: Features to write (assumed WGS84).+ * layerName: Name of the single feature table / layer in the GeoPackage.+ *+ * Returns:+ * The GeoPackage file as bytes.+ */+export async function writeGeoPackage(+ geojson: FeatureCollection,+ layerName: string+): Promise<Uint8Array> {+ const SQL = await loadSqlJs();+ return writeGeoPackageSync(SQL, geojson, layerName);+}diff --git a/src/processors/src/export/geoparquet-writer.ts b/src/processors/src/export/geoparquet-writer.tsnew file mode 100644index 000000000..9a2dee705--- /dev/null+++ b/src/processors/src/export/geoparquet-writer.ts@@ -0,0 +1,179 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {convertGeometryToWKB} from '@loaders.gl/gis';+import * as arrow from 'apache-arrow';+import type {FeatureCollection, Geometry} from 'geojson';++const GEOPARQUET_GEOMETRY_TYPES = new Set([+ 'Point',+ 'LineString',+ 'Polygon',+ 'MultiPoint',+ 'MultiLineString',+ 'MultiPolygon',+ 'GeometryCollection'+]);++type ParquetWasm = typeof import('parquet-wasm');++/** Keep in sync with the parquet-wasm dependency in package.json. */+const PARQUET_WASM_URL = 'https://unpkg.com/[email protected]/esm/parquet_wasm_bg.wasm';++let parquetWasmPromise: Promise<ParquetWasm> | null = null;++async function importParquetWasm(): Promise<ParquetWasm> {+ // Jest uses the CommonJS build, which reads the wasm from disk. A literal+ // import of that build makes esbuild pull in Node's fs/path. The browser+ // bundle only keeps the ESM build below, because NODE_ENV is not "test".+ if (process.env.NODE_ENV === 'test') {+ const specifier = 'parquet-wasm/node';+ const imported = (await import(specifier)) as ParquetWasm;+ return (imported.Table ? imported : imported.default) as ParquetWasm;+ }+ const imported = await import('parquet-wasm');+ if (typeof imported.default === 'function') {+ await imported.default(PARQUET_WASM_URL);+ }+ return (imported.Table ? imported : imported.default) as ParquetWasm;+}++function loadParquetWasm(): Promise<ParquetWasm> {+ if (parquetWasmPromise) {+ return parquetWasmPromise;+ }+ const pending = importParquetWasm().catch(error => {+ parquetWasmPromise = null;+ throw error;+ });+ parquetWasmPromise = pending;+ return pending;+}++function wkbBytes(geometry: Geometry | null): Uint8Array | null {+ if (!geometry) {+ return null;+ }+ return new Uint8Array(convertGeometryToWKB(geometry));+}++function propertyVector(values: (string | number | boolean | null)[]): arrow.Vector {+ // tableFromArrays stores JS strings as Dictionary<Int32, Utf8>. Parquet import+ // rejects that type with "arrow type not supported: Dictionary".+ let kind: 'bool' | 'float' | 'string' = 'string';+ let seen = false;+ for (const value of values) {+ if (value == null) {+ continue;+ }+ let next: 'bool' | 'float' | 'string' = 'string';+ if (typeof value === 'boolean') {+ next = 'bool';+ } else if (typeof value === 'number') {+ next = 'float';+ }+ if (!seen) {+ kind = next;+ seen = true;+ } else if (kind !== next) {+ kind = 'string';+ break;+ }+ }+ if (kind === 'bool') {+ return arrow.vectorFromArray(values as (boolean | null)[], new arrow.Bool());+ }+ if (kind === 'float') {+ return arrow.vectorFromArray(values as (number | null)[], new arrow.Float64());+ }+ return arrow.vectorFromArray(+ values.map(value => (value == null ? null : String(value))),+ new arrow.Utf8()+ );+}++function parquetValue(value: unknown): string | number | boolean | null {+ if (value == null) {+ return null;+ }+ if (typeof value === 'number') {+ return Number.isFinite(value) ? value : null;+ }+ if (typeof value === 'boolean' || typeof value === 'string') {+ return value;+ }+ return JSON.stringify(value);+}++function propertyName(name: string, taken: Set<string>): string {+ let candidate = name === 'geometry' ? 'geometry_attr' : name;+ if (!candidate) {+ candidate = 'field';+ }+ let unique = candidate;+ let index = 1;+ while (taken.has(unique)) {+ unique = `${candidate}_${index}`;+ index += 1;+ }+ taken.add(unique);+ return unique;+}++/**+ * Write a GeoJSON FeatureCollection as GeoParquet 1.1 (WKB geometry, WGS84).+ */+export async function writeGeoParquet(geojson: FeatureCollection): Promise<Uint8Array> {+ const features = geojson.features ?? [];+ const taken = new Set<string>(['geometry']);+ const propertyKeys: {source: string; name: string}[] = [];+ const seen = new Set<string>();+ for (const feature of features) {+ for (const key of Object.keys(feature.properties ?? {})) {+ if (!seen.has(key)) {+ seen.add(key);+ propertyKeys.push({source: key, name: propertyName(key, taken)});+ }+ }+ }++ const columns: Record<string, arrow.Vector> = {+ geometry: arrow.vectorFromArray(+ features.map(feature => wkbBytes(feature.geometry)),+ new arrow.Binary()+ )+ };+ for (const key of propertyKeys) {+ columns[key.name] = propertyVector(+ features.map(feature => parquetValue(feature.properties?.[key.source]))+ );+ }++ const table = new arrow.Table(columns);+ const ipc = arrow.tableToIPC(table, 'stream');+ const parquetWasm = await loadParquetWasm();+ const wasmTable = parquetWasm.Table.fromIPCStream(ipc);+ const geometryTypes: string[] = [];+ for (const feature of features) {+ const type = feature.geometry?.type;+ if (type && GEOPARQUET_GEOMETRY_TYPES.has(type) && !geometryTypes.includes(type)) {+ geometryTypes.push(type);+ }+ }+ const geoMetadata = JSON.stringify({+ version: '1.1.0',+ primary_column: 'geometry',+ columns: {+ geometry: {+ encoding: 'WKB',+ ...(geometryTypes.length ? {geometry_types: geometryTypes} : {})+ }+ }+ });+ const properties = new parquetWasm.WriterPropertiesBuilder()+ .setKeyValueMetadata(new Map([['geo', geoMetadata]]))+ .build();+ // writeParquet takes ownership of both objects.+ const bytes = parquetWasm.writeParquet(wasmTable, properties);+ return new Uint8Array(bytes);+}diff --git a/src/processors/src/export/kml-writer.ts b/src/processors/src/export/kml-writer.tsnew file mode 100644index 000000000..99c7f73d5--- /dev/null+++ b/src/processors/src/export/kml-writer.ts@@ -0,0 +1,318 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project+// Adapted from GeoLibre (https://github.com/opengeos/GeoLibre), MIT License.++import type {Feature, FeatureCollection, GeoJsonProperties, Geometry, Position} from 'geojson';+export class KmlCoordinateError extends Error {+ constructor(featureIndex: number, featureId?: string | number) {+ super(`Feature ${featureId ?? featureIndex} has a coordinate that KML cannot represent.`);+ this.name = 'KmlCoordinateError';+ }+}++const KML_NAMESPACE = 'http://www.opengis.net/kml/2.2';++class InvalidCoordinateError extends Error {}++function xmlSafeText(value: unknown): string {+ return stripDisallowedControls(String(value))+ .replace(/&/g, '&')+ .replace(/</g, '<')+ .replace(/>/g, '>')+ .replace(/"/g, '"')+ .replace(/'/g, ''');+}++/** XML 1.0 disallows these controls. Tab, newline, and carriage return stay. */+function stripDisallowedControls(value: string): string {+ let result = '';+ for (const char of value) {+ const code = char.charCodeAt(0);+ const disallowed =+ code <= 0x08 || code === 0x0b || code === 0x0c || (code >= 0x0e && code <= 0x1f);+ result += disallowed ? '\uFFFD' : char;+ }+ return result;+}++function indentXml(xml: string, spaces: number): string {+ const indentation = ' '.repeat(spaces);+ return xml+ .split('\n')+ .map(line => `${indentation}${line}`)+ .join('\n');+}++function decimalText(value: number): string {+ const text = String(value);+ const match = /^(-?)(\d+)(?:\.(\d+))?e([+-]?\d+)$/i.exec(text);+ if (!match) return text;++ const [, sign, integer, fraction = '', rawExponent] = match;+ const digits = `${integer}${fraction}`;+ const decimalIndex = integer.length + Number(rawExponent);+ if (decimalIndex <= 0) {+ return `${sign}0.${'0'.repeat(-decimalIndex)}${digits}`;+ }+ if (decimalIndex >= digits.length) {+ return `${sign}${digits}${'0'.repeat(decimalIndex - digits.length)}`;+ }+ return `${sign}${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`;+}++function positionText(position: Position): string {+ if (position.length < 2 || position.slice(0, 3).some(value => !Number.isFinite(value))) {+ throw new InvalidCoordinateError();+ }+ return position+ .slice(0, 3)+ .map(value => decimalText(value))+ .join(',');+}++function coordinatesText(positions: Position[]): string {+ return positions.map(positionText).join(' ');+}++function hasAltitude(position: Position): boolean {+ // KML applies altitude mode to the entire geometry. For mixed-dimensional+ // GeoJSON, preserve supplied altitudes and let KML interpret 2D vertices as+ // having its implicit zero altitude.+ return position.length >= 3;+}++function closedRing(ring: Position[]): Position[] {+ if (ring.length === 0) return ring;+ const first = ring[0];+ const last = ring[ring.length - 1];+ if (+ first.length === last.length &&+ first.every((coordinate, index) => coordinate === last[index])+ ) {+ return ring;+ }+ return [...ring, first];+}++function polygonKml(coordinates: Position[][]): string {+ const [outer, ...inner] = coordinates;+ const boundaries: string[] = [];+ if (outer) {+ boundaries.push(+ '<outerBoundaryIs>',+ ' <LinearRing>',+ ` <coordinates>${coordinatesText(closedRing(outer))}</coordinates>`,+ ' </LinearRing>',+ '</outerBoundaryIs>'+ );+ }+ for (const ring of inner) {+ boundaries.push(+ '<innerBoundaryIs>',+ ' <LinearRing>',+ ` <coordinates>${coordinatesText(closedRing(ring))}</coordinates>`,+ ' </LinearRing>',+ '</innerBoundaryIs>'+ );+ }+ if (boundaries.length === 0) return '<Polygon />';+ const contents = coordinates.some(ring => ring.some(hasAltitude))+ ? ['<altitudeMode>absolute</altitudeMode>', ...boundaries]+ : boundaries;+ return `<Polygon>\n${indentXml(contents.join('\n'), 2)}\n</Polygon>`;+}++function multiGeometryKml(geometries: Geometry[]): string {+ if (geometries.length === 0) return '<MultiGeometry />';+ return `<MultiGeometry>\n${geometries+ .map(geometry => indentXml(geometryKml(geometry), 2))+ .join('\n')}\n</MultiGeometry>`;+}++function geometryKml(geometry: Geometry): string {+ switch (geometry.type) {+ case 'Point':+ return `<Point>${+ hasAltitude(geometry.coordinates) ? '<altitudeMode>absolute</altitudeMode>' : ''+ }<coordinates>${positionText(geometry.coordinates)}</coordinates></Point>`;+ case 'MultiPoint':+ return multiGeometryKml(+ geometry.coordinates.map(coordinates => ({type: 'Point', coordinates}))+ );+ case 'LineString':+ return `<LineString>${+ geometry.coordinates.some(hasAltitude) ? '<altitudeMode>absolute</altitudeMode>' : ''+ }<coordinates>${coordinatesText(geometry.coordinates)}</coordinates></LineString>`;+ case 'MultiLineString':+ return multiGeometryKml(+ geometry.coordinates.map(coordinates => ({type: 'LineString', coordinates}))+ );+ case 'Polygon':+ return polygonKml(geometry.coordinates);+ case 'MultiPolygon':+ return multiGeometryKml(+ geometry.coordinates.map(coordinates => ({type: 'Polygon', coordinates}))+ );+ case 'GeometryCollection':+ return multiGeometryKml(geometry.geometries);+ }+}++function propertyText(value: unknown): string {+ if (value == null) return '';+ if (typeof value !== 'object') return String(value);+ return JSON.stringify(value);+}++function propertyElement(properties: GeoJsonProperties, key: string): string | null {+ const value = properties?.[key];+ return value == null || typeof value === 'object'+ ? null+ : `<${key}>${xmlSafeText(value)}</${key}>`;+}++function extendedDataKml(feature: Feature): string | null {+ const properties = feature.properties ?? {};+ const entries = Object.entries(properties).filter(+ ([key, value]) =>+ !((key === 'name' || key === 'description') && value != null && typeof value !== 'object')+ );+ if (feature.id != null) {+ let idKey = 'feature_id';+ if (Object.prototype.hasOwnProperty.call(properties, idKey)) {+ idKey = 'geojson_feature_id';+ let suffix = 2;+ while (Object.prototype.hasOwnProperty.call(properties, idKey)) {+ idKey = `geojson_feature_id_${suffix}`;+ suffix += 1;+ }+ }+ entries.unshift([idKey, feature.id]);+ }+ if (entries.length === 0) return null;+ const data = entries.map(+ ([key, value]) =>+ `<Data name="${xmlSafeText(key)}"><value>${xmlSafeText(propertyText(value))}</value></Data>`+ );+ return `<ExtendedData>\n${data.map(entry => indentXml(entry, 2)).join('\n')}\n</ExtendedData>`;+}++function opacityByte(value: unknown, defaultOpacity = 1): string {+ const opacity = typeof value === 'number' ? value : Number(value ?? defaultOpacity);+ const normalized = Number.isFinite(opacity) ? Math.min(1, Math.max(0, opacity)) : defaultOpacity;+ return Math.round(normalized * 255)+ .toString(16)+ .padStart(2, '0');+}++function kmlColor(color: unknown, opacity: unknown, defaultOpacity = 1): string | null {+ if (typeof color !== 'string') return null;+ const match = /^#?([0-9a-f]{3}|[0-9a-f]{6})$/i.exec(color.trim());+ if (!match) return null;+ const hex =+ match[1].length === 3+ ? match[1]+ .split('')+ .map(digit => `${digit}${digit}`)+ .join('')+ : match[1];+ const red = hex.slice(0, 2);+ const green = hex.slice(2, 4);+ const blue = hex.slice(4, 6);+ return `${opacityByte(opacity, defaultOpacity)}${blue}${green}${red}`.toLowerCase();+}++function styleKml(properties: GeoJsonProperties): string | null {+ if (!properties) return null;+ const sections: string[] = [];++ const markerColor = kmlColor(properties['marker-color'], properties['marker-opacity']);+ if (markerColor) {+ sections.push(`<IconStyle><color>${markerColor}</color></IconStyle>`);+ }++ const strokeColor = kmlColor(properties.stroke, properties['stroke-opacity']);+ const strokeWidth = Number(properties['stroke-width']);+ if (strokeColor || Number.isFinite(strokeWidth)) {+ const values = [+ strokeColor ? `<color>${strokeColor}</color>` : null,+ Number.isFinite(strokeWidth) ? `<width>${strokeWidth}</width>` : null+ ].filter((value): value is string => value !== null);+ sections.push(`<LineStyle>${values.join('')}</LineStyle>`);+ }++ const fillColor = kmlColor(properties.fill, properties['fill-opacity'], 0.6);+ if (fillColor) {+ sections.push(`<PolyStyle><color>${fillColor}</color></PolyStyle>`);+ }++ if (sections.length === 0) return null;+ return `<Style>${sections.join('')}</Style>`;+}++function featureKml(feature: Feature, style: string | null): string {+ const properties = feature.properties ?? {};+ const contents = [+ propertyElement(properties, 'name'),+ propertyElement(properties, 'description'),+ style,+ extendedDataKml(feature),+ feature.geometry ? geometryKml(feature.geometry) : null+ ].filter((value): value is string => value !== null);++ if (contents.length === 0) return '<Placemark />';+ return `<Placemark>\n${contents.map(value => indentXml(value, 2)).join('\n')}\n</Placemark>`;+}++/**+ * Convert a GeoJSON FeatureCollection to a KML 2.2 document.+ *+ * Feature properties are retained in ExtendedData, except primitive `name` and+ * `description` values that are represented by their native KML elements.+ * Simplestyle properties are also promoted to native KML styles for display in+ * Google Earth and other KML clients.+ */+export function writeKml(geojson: FeatureCollection, documentName: string): string {+ const featureStyles = geojson.features.map(feature => styleKml(feature.properties));+ const styleCounts = new Map<string, number>();+ for (const style of featureStyles) {+ if (style) styleCounts.set(style, (styleCounts.get(style) ?? 0) + 1);+ }++ const sharedStyles = new Map<string, string>();+ for (const [style, count] of styleCounts) {+ if (count > 1) sharedStyles.set(style, `style-${sharedStyles.size + 1}`);+ }+ const styleDefinitions = Array.from(sharedStyles, ([style, id]) =>+ indentXml(style.replace('<Style>', `<Style id="${id}">`), 4)+ );++ const placemarks = geojson.features.map((feature, index) => {+ try {+ const style = featureStyles[index];+ const sharedStyleId = style ? sharedStyles.get(style) : undefined;+ return indentXml(+ featureKml(feature, sharedStyleId ? `<styleUrl>#${sharedStyleId}</styleUrl>` : style),+ 4+ );+ } catch (error) {+ if (error instanceof InvalidCoordinateError) {+ throw new KmlCoordinateError(index, feature.id);+ }+ throw error;+ }+ });+ const documentContents = [+ ` <name>${xmlSafeText(documentName)}</name>`,+ ...styleDefinitions,+ ...placemarks+ ].join('\n');+ return [+ '<?xml version="1.0" encoding="UTF-8"?>',+ `<kml xmlns="${KML_NAMESPACE}">`,+ ' <Document>',+ documentContents,+ ' </Document>',+ '</kml>'+ ].join('\n');+}diff --git a/src/processors/src/export/shapefile-writer.ts b/src/processors/src/export/shapefile-writer.tsnew file mode 100644index 000000000..5990c350a--- /dev/null+++ b/src/processors/src/export/shapefile-writer.ts@@ -0,0 +1,510 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project+// Adapted from GeoLibre (https://github.com/opengeos/GeoLibre), MIT License.++import type {Feature, FeatureCollection, Geometry, Position} from 'geojson';++/**+ * Write a GeoJSON FeatureCollection to an ESRI Shapefile dataset (the .shp,+ * .shx, .dbf, .prj, and .cpg parts) entirely in the browser.+ *+ * DuckDB-WASM and Pyodide's bundled GDAL cannot write GDAL vector formats in the+ * browser, so the binary parts are assembled directly. The dataset is WGS84+ * (EPSG:4326). All non-null geometries must share one shape family+ * (point/line/polygon). Null geometries are written as the Shapefile "Null"+ * shape so the record count still lines up with the .dbf.+ */++export interface ShapefileParts {+ shp: Uint8Array;+ shx: Uint8Array;+ dbf: Uint8Array;+ prj: Uint8Array;+ cpg: Uint8Array;+}++const SHAPE_NULL = 0;+const SHAPE_POINT = 1;+const SHAPE_POLYLINE = 3;+const SHAPE_POLYGON = 5;+const SHAPE_MULTIPOINT = 8;++const WGS84_PRJ =+ 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",' ++ '6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],' ++ 'UNIT["Degree",0.0174532925199433]]';++type Family = 'point' | 'line' | 'polygon';++function familyOf(geometry: Geometry): Family | null {+ switch (geometry.type) {+ case 'Point':+ case 'MultiPoint':+ return 'point';+ case 'LineString':+ case 'MultiLineString':+ return 'line';+ case 'Polygon':+ case 'MultiPolygon':+ return 'polygon';+ default:+ return null;+ }+}++/** Pick the single Shapefile shape type for the whole file. */+function fileShapeType(features: Feature[]): {type: number; family: Family} {+ let family: Family | null = null;+ let hasMultiPoint = false;+ for (const feature of features) {+ if (!feature.geometry) continue;+ const geomFamily = familyOf(feature.geometry);+ if (!geomFamily) {+ throw new Error(`Shapefile export does not support ${feature.geometry.type} geometries.`);+ }+ if (family !== null && family !== geomFamily) {+ throw new Error('Shapefile export requires all geometries to have the same shape family.');+ }+ if (family === null) family = geomFamily;+ if (feature.geometry.type === 'MultiPoint') hasMultiPoint = true;+ }+ if (family === null) {+ throw new Error('The layer has no supported geometries to export.');+ }+ if (family === 'point') {+ return {type: hasMultiPoint ? SHAPE_MULTIPOINT : SHAPE_POINT, family};+ }+ return {type: family === 'line' ? SHAPE_POLYLINE : SHAPE_POLYGON, family};+}++/** Signed area of a ring; positive is counter-clockwise (screen coords, y up). */+function ringSignedArea(ring: Position[]): number {+ let area = 0;+ for (let i = 0; i < ring.length - 1; i += 1) {+ const [x1, y1] = ring[i];+ const [x2, y2] = ring[i + 1];+ area += Number(x1) * Number(y2) - Number(x2) * Number(y1);+ }+ return area / 2;+}++/** Return a ring oriented clockwise (`wantClockwise`) or counter-clockwise.+ * Shapefile polygons require clockwise outer rings and counter-clockwise holes. */+function orientRing(ring: Position[], wantClockwise: boolean): Position[] {+ const isClockwise = ringSignedArea(ring) < 0;+ return isClockwise === wantClockwise ? ring : [...ring].reverse();+}++/** Flatten a geometry into Shapefile "parts" (rings/lines) of positions, with+ * polygon rings reoriented for the Shapefile convention. */+function geometryParts(geometry: Geometry, family: Family): Position[][] {+ if (family === 'line') {+ if (geometry.type === 'LineString') return [geometry.coordinates];+ if (geometry.type === 'MultiLineString') return geometry.coordinates;+ return [];+ }+ // polygon+ const polygons =+ geometry.type === 'Polygon'+ ? [geometry.coordinates]+ : geometry.type === 'MultiPolygon'+ ? geometry.coordinates+ : [];+ const parts: Position[][] = [];+ for (const rings of polygons) {+ rings.forEach((ring, index) => {+ // First ring of each polygon is the outer ring (clockwise); the rest are+ // holes (counter-clockwise).+ parts.push(orientRing(ring, index === 0));+ });+ }+ return parts;+}++function pointsOf(geometry: Geometry): Position[] {+ if (geometry.type === 'Point') return [geometry.coordinates];+ if (geometry.type === 'MultiPoint') return geometry.coordinates;+ return [];+}++interface Box {+ minX: number;+ minY: number;+ maxX: number;+ maxY: number;+}++function emptyBox(): Box {+ return {minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity};+}++function extend(box: Box, x: number, y: number): void {+ if (Number.isFinite(x)) {+ box.minX = Math.min(box.minX, x);+ box.maxX = Math.max(box.maxX, x);+ }+ if (Number.isFinite(y)) {+ box.minY = Math.min(box.minY, y);+ box.maxY = Math.max(box.maxY, y);+ }+}++function safeBox(box: Box): Box {+ return Number.isFinite(box.minX) ? box : {minX: 0, minY: 0, maxX: 0, maxY: 0};+}++/** Encode one geometry's Shapefile record content (without the 8-byte record+ * header). Returns the content bytes and accumulates into the file bbox. */+function encodeShape(+ geometry: Geometry | null,+ shapeType: number,+ family: Family,+ fileBox: Box+): Uint8Array {+ if (!geometry || familyOf(geometry) !== family) {+ // Null shape: just the int32 little-endian shape type 0.+ const nullShape = new Uint8Array(4);+ new DataView(nullShape.buffer).setInt32(0, SHAPE_NULL, true);+ return nullShape;+ }++ if (shapeType === SHAPE_POINT) {+ const [x, y] = pointsOf(geometry)[0] ?? [0, 0];+ extend(fileBox, Number(x), Number(y));+ const content = new Uint8Array(20);+ const view = new DataView(content.buffer);+ view.setInt32(0, SHAPE_POINT, true);+ view.setFloat64(4, Number(x), true);+ view.setFloat64(12, Number(y), true);+ return content;+ }++ if (shapeType === SHAPE_MULTIPOINT) {+ const points = pointsOf(geometry);+ const box = emptyBox();+ for (const [x, y] of points) extend(box, Number(x), Number(y));+ const drawn = safeBox(box);+ extend(fileBox, drawn.minX, drawn.minY);+ extend(fileBox, drawn.maxX, drawn.maxY);+ const content = new Uint8Array(4 + 32 + 4 + points.length * 16);+ const view = new DataView(content.buffer);+ view.setInt32(0, SHAPE_MULTIPOINT, true);+ view.setFloat64(4, drawn.minX, true);+ view.setFloat64(12, drawn.minY, true);+ view.setFloat64(20, drawn.maxX, true);+ view.setFloat64(28, drawn.maxY, true);+ view.setInt32(36, points.length, true);+ let offset = 40;+ for (const [x, y] of points) {+ view.setFloat64(offset, Number(x), true);+ view.setFloat64(offset + 8, Number(y), true);+ offset += 16;+ }+ return content;+ }++ // PolyLine (3) or Polygon (5): identical record layout.+ const parts = geometryParts(geometry, family);+ const numParts = parts.length;+ const flatPoints: Position[] = parts.flat();+ const box = emptyBox();+ for (const [x, y] of flatPoints) extend(box, Number(x), Number(y));+ const drawn = safeBox(box);+ extend(fileBox, drawn.minX, drawn.minY);+ extend(fileBox, drawn.maxX, drawn.maxY);++ const content = new Uint8Array(4 + 32 + 4 + 4 + numParts * 4 + flatPoints.length * 16);+ const view = new DataView(content.buffer);+ view.setInt32(0, shapeType, true);+ view.setFloat64(4, drawn.minX, true);+ view.setFloat64(12, drawn.minY, true);+ view.setFloat64(20, drawn.maxX, true);+ view.setFloat64(28, drawn.maxY, true);+ view.setInt32(36, numParts, true);+ view.setInt32(40, flatPoints.length, true);+ let offset = 44;+ let runningIndex = 0;+ for (const part of parts) {+ view.setInt32(offset, runningIndex, true);+ offset += 4;+ runningIndex += part.length;+ }+ for (const [x, y] of flatPoints) {+ view.setFloat64(offset, Number(x), true);+ view.setFloat64(offset + 8, Number(y), true);+ offset += 16;+ }+ return content;+}++/** Write the 100-byte .shp/.shx header. `fileWords` is the file length in+ * 16-bit words; `box` the dataset bounding box. */+function writeMainHeader(target: Uint8Array, shapeType: number, fileWords: number, box: Box): void {+ const view = new DataView(target.buffer, target.byteOffset, 100);+ view.setInt32(0, 9994, false); // file code (big-endian)+ view.setInt32(24, fileWords, false); // file length in words (big-endian)+ view.setInt32(28, 1000, true); // version (little-endian)+ view.setInt32(32, shapeType, true);+ const drawn = safeBox(box);+ view.setFloat64(36, drawn.minX, true);+ view.setFloat64(44, drawn.minY, true);+ view.setFloat64(52, drawn.maxX, true);+ view.setFloat64(60, drawn.maxY, true);+ // Z and M ranges remain zero (2D output).+}++// ----- DBF (attribute table) -----++type DbfFieldType = 'C' | 'N' | 'L';++interface DbfField {+ key: string;+ name: string;+ type: DbfFieldType;+ length: number;+ decimals: number;+}++const TEXT_ENCODER = new TextEncoder();++/** Encode `text` to UTF-8, truncated to at most `maxBytes` without splitting a+ * multi-byte code point (so non-ASCII attribute values stay valid). */+function encodeUtf8Truncated(text: string, maxBytes: number): Uint8Array {+ const full = TEXT_ENCODER.encode(text);+ if (full.length <= maxBytes) return full;+ let length = 0;+ for (const char of text) {+ const size = TEXT_ENCODER.encode(char).length;+ if (length + size > maxBytes) break;+ length += size;+ }+ return full.subarray(0, length);+}++function dbfFieldName(key: string, taken: Set<string>): string {+ // DBF field names are at most 10 characters; truncate and de-duplicate.+ const base = key.replace(/[^0-9A-Za-z_]/g, '_').slice(0, 10) || 'field';+ if (!taken.has(base.toUpperCase())) {+ taken.add(base.toUpperCase());+ return base;+ }+ for (let i = 1; i < 1000; i += 1) {+ const suffix = String(i);+ const candidate = `${base.slice(0, 10 - suffix.length)}${suffix}`;+ if (!taken.has(candidate.toUpperCase())) {+ taken.add(candidate.toUpperCase());+ return candidate;+ }+ }+ // Falling through means >999 fields share one 10-char prefix; returning a+ // name already in `taken` would write a duplicate DBF column. Fail loudly+ // instead of silently corrupting the file.+ throw new Error(`Cannot generate a unique DBF field name for "${key}" (>999 collisions).`);+}++function decimalPlaces(value: number): number {+ if (Number.isInteger(value)) return 0;+ const text = String(value);+ const dot = text.indexOf('.');+ if (dot < 0) return 0;+ return Math.min(15, text.length - dot - 1);+}++function planField(key: string, name: string, values: unknown[]): DbfField {+ let sawNumber = false;+ let sawNonNumber = false;+ let sawBoolean = false;+ let sawNonBoolean = false;+ for (const value of values) {+ if (value == null) continue;+ if (typeof value === 'boolean') {+ sawBoolean = true;+ } else {+ sawNonBoolean = true;+ }+ if (typeof value === 'number' && Number.isFinite(value)) {+ sawNumber = true;+ } else {+ sawNonNumber = true;+ }+ }++ if (sawBoolean && !sawNonBoolean) {+ return {key, name, type: 'L', length: 1, decimals: 0};+ }++ if (sawNumber && !sawNonNumber) {+ let decimals = 0;+ for (const value of values) {+ if (typeof value === 'number' && Number.isFinite(value)) {+ decimals = Math.max(decimals, decimalPlaces(value));+ }+ }+ let length = 1;+ for (const value of values) {+ if (typeof value !== 'number' || !Number.isFinite(value)) continue;+ const text = decimals > 0 ? value.toFixed(decimals) : String(value);+ length = Math.max(length, text.length);+ }+ return {key, name, type: 'N', length: Math.min(length, 33), decimals};+ }++ // Fall back to text for everything else (strings, mixed types, objects).+ let length = 1;+ for (const value of values) {+ if (value == null) continue;+ const text = typeof value === 'object' ? JSON.stringify(value) : String(value);+ length = Math.max(length, TEXT_ENCODER.encode(text).length);+ }+ return {key, name, type: 'C', length: Math.min(length, 254), decimals: 0};+}++function encodeDbfValue(value: unknown, field: DbfField): Uint8Array {+ const cell = new Uint8Array(field.length).fill(0x20); // space-padded+ if (value == null) return cell;++ if (field.type === 'L') {+ cell[0] = value ? 0x54 : 0x46; // 'T' / 'F'+ return cell;+ }++ if (field.type === 'N') {+ if (typeof value !== 'number' || !Number.isFinite(value)) return cell;+ let text = field.decimals > 0 ? value.toFixed(field.decimals) : String(value);+ if (text.length > field.length) text = text.slice(0, field.length);+ // Numbers are right-justified in DBF.+ const bytes = TEXT_ENCODER.encode(text);+ cell.set(bytes, field.length - bytes.length);+ return cell;+ }++ // Text: left-justified, truncated to the field byte length on a code-point+ // boundary so multi-byte characters are never cut mid-sequence.+ const text = typeof value === 'object' ? JSON.stringify(value) : String(value);+ cell.set(encodeUtf8Truncated(text, field.length), 0);+ return cell;+}++function buildDbf(features: Feature[], fields: DbfField[]): Uint8Array {+ const recordLength = 1 + fields.reduce((total, field) => total + field.length, 0);+ const headerLength = 32 + fields.length * 32 + 1;+ // Both are serialized as uint16; too many or too-wide attributes would+ // overflow and silently corrupt the .dbf, so fail fast instead.+ if (recordLength > 0xffff || headerLength > 0xffff) {+ throw new Error('Too many or too-wide attributes for the DBF format limits.');+ }+ const total = headerLength + features.length * recordLength + 1; // +1 EOF+ const buffer = new Uint8Array(total);+ const view = new DataView(buffer.buffer);++ buffer[0] = 0x03; // dBASE III without memo+ const now = new Date();+ buffer[1] = now.getFullYear() - 1900;+ buffer[2] = now.getMonth() + 1;+ buffer[3] = now.getDate();+ view.setUint32(4, features.length, true);+ view.setUint16(8, headerLength, true);+ view.setUint16(10, recordLength, true);++ let offset = 32;+ for (const field of fields) {+ const nameBytes = TEXT_ENCODER.encode(field.name).subarray(0, 11);+ buffer.set(nameBytes, offset); // remaining name bytes stay 0 (null-padded)+ buffer[offset + 11] = field.type.charCodeAt(0);+ buffer[offset + 16] = field.length;+ buffer[offset + 17] = field.decimals;+ offset += 32;+ }+ buffer[offset] = 0x0d; // field descriptor terminator+ offset += 1;++ for (const feature of features) {+ buffer[offset] = 0x20; // record not deleted+ offset += 1;+ for (const field of fields) {+ const cell = encodeDbfValue(feature.properties?.[field.key], field);+ buffer.set(cell, offset);+ offset += field.length;+ }+ }+ buffer[offset] = 0x1a; // end-of-file marker+ return buffer;+}++/**+ * Assemble the parts of an ESRI Shapefile from a FeatureCollection.+ *+ * Args:+ * geojson: Features to write (assumed WGS84).+ *+ * Returns:+ * The .shp/.shx/.dbf/.prj/.cpg byte buffers (the caller names and zips them).+ */+export function writeShapefile(geojson: FeatureCollection): ShapefileParts {+ const features = geojson.features ?? [];+ if (features.length === 0) {+ throw new Error('The layer has no features to export.');+ }+ const {type: shapeType, family} = fileShapeType(features);++ const fileBox = emptyBox();+ const contents = features.map(feature =>+ encodeShape(feature.geometry, shapeType, family, fileBox)+ );++ // .shp: 100-byte header + per-record (8-byte header + content).+ const shpSize = 100 + contents.reduce((total, content) => total + 8 + content.length, 0);+ const shp = new Uint8Array(shpSize);+ const shpView = new DataView(shp.buffer);+ const shxSize = 100 + contents.length * 8;+ const shx = new Uint8Array(shxSize);+ const shxView = new DataView(shx.buffer);++ writeMainHeader(shp, shapeType, shpSize / 2, fileBox);+ writeMainHeader(shx, shapeType, shxSize / 2, fileBox);++ let shpOffset = 100;+ let shxOffset = 100;+ contents.forEach((content, index) => {+ const contentWords = content.length / 2;+ const recordOffsetWords = shpOffset / 2;+ // Record header (big-endian): 1-based record number + content length.+ shpView.setInt32(shpOffset, index + 1, false);+ shpView.setInt32(shpOffset + 4, contentWords, false);+ shp.set(content, shpOffset + 8);+ shpOffset += 8 + content.length;+ // .shx entry (big-endian): record offset + content length, both in words.+ shxView.setInt32(shxOffset, recordOffsetWords, false);+ shxView.setInt32(shxOffset + 4, contentWords, false);+ shxOffset += 8;+ });++ // DBF fields from the union of property keys across all features.+ const keys: string[] = [];+ const seen = new Set<string>();+ for (const feature of features) {+ for (const key of Object.keys(feature.properties ?? {})) {+ if (!seen.has(key)) {+ seen.add(key);+ keys.push(key);+ }+ }+ }+ const takenNames = new Set<string>();+ const fields = keys.map(key =>+ planField(+ key,+ dbfFieldName(key, takenNames),+ features.map(feature => feature.properties?.[key])+ )+ );+ const dbf = buildDbf(features, fields);++ return {+ shp,+ shx,+ dbf,+ prj: TEXT_ENCODER.encode(WGS84_PRJ),+ cpg: TEXT_ENCODER.encode('UTF-8')+ };+}diff --git a/src/processors/src/export/sql-js.spec.ts b/src/processors/src/export/sql-js.spec.tsnew file mode 100644index 000000000..9437ed841--- /dev/null+++ b/src/processors/src/export/sql-js.spec.ts@@ -0,0 +1,28 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {loadSqlJs} from './sql-js';++describe('loadSqlJs', () => {+ test('uses the package loader in a production Node environment', async () => {+ const previousNodeEnv = process.env.NODE_ENV;+ const documentDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'document');+ process.env.NODE_ENV = 'production';+ Object.defineProperty(globalThis, 'document', {configurable: true, value: undefined});+ try {+ expect(typeof document).toBe('undefined');+ const SQL = await loadSqlJs();+ const db = new SQL.Database();+ db.run('CREATE TABLE places (name TEXT);');+ expect(db.exec("SELECT name FROM sqlite_master WHERE name = 'places'")[0].values).toEqual([+ ['places']+ ]);+ db.close();+ } finally {+ process.env.NODE_ENV = previousNodeEnv;+ if (documentDescriptor) {+ Object.defineProperty(globalThis, 'document', documentDescriptor);+ }+ }+ });+});diff --git a/src/processors/src/export/sql-js.ts b/src/processors/src/export/sql-js.tsnew file mode 100644index 000000000..8e0b16ca2--- /dev/null+++ b/src/processors/src/export/sql-js.ts@@ -0,0 +1,63 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import type {SqlJsStatic} from 'sql.js';++/** Keep in sync with the sql.js dependency in package.json. */+const SQL_JS_VERSION = '1.13.0';+const SQL_JS_CDN = `https://cdn.jsdelivr.net/npm/sql.js@${SQL_JS_VERSION}/dist/`;++let sqlPromise: Promise<SqlJsStatic> | null = null;++type InitSqlJs = (config?: {locateFile?: (file: string) => string}) => Promise<SqlJsStatic>;++/**+ * Load sql.js once.+ * Tests import the package, which reads the wasm from disk. The browser build+ * of that package calls Node's fs/path/crypto, so the demo bundle loads the+ * script from a CDN instead of importing it.+ */+export function loadSqlJs(): Promise<SqlJsStatic> {+ if (!sqlPromise) {+ const pending = (+ process.env.NODE_ENV === 'test' || typeof document === 'undefined'+ ? loadSqlJsFromPackage()+ : loadSqlJsFromCdn()+ ).catch(error => {+ sqlPromise = null;+ throw error;+ });+ sqlPromise = pending;+ }+ return sqlPromise;+}++async function loadSqlJsFromPackage(): Promise<SqlJsStatic> {+ // Non-literal so browser bundlers do not pull in sql-wasm.js (it requires fs).+ const specifier = 'sql.js';+ const sqlModule = (await import(specifier)) as {default: InitSqlJs};+ return sqlModule.default();+}++function loadSqlJsFromCdn(): Promise<SqlJsStatic> {+ const locateFile = (file: string) => `${SQL_JS_CDN}${file}`;+ const existing = (globalThis as {initSqlJs?: InitSqlJs}).initSqlJs;+ if (existing) {+ return existing({locateFile});+ }+ return new Promise((resolve, reject) => {+ const script = document.createElement('script');+ script.src = `${SQL_JS_CDN}sql-wasm.js`;+ script.async = true;+ script.onload = () => {+ const init = (globalThis as {initSqlJs?: InitSqlJs}).initSqlJs;+ if (!init) {+ reject(new Error('sql.js failed to initialize'));+ return;+ }+ resolve(init({locateFile}));+ };+ script.onerror = () => reject(new Error('Failed to load sql.js'));+ document.head.appendChild(script);+ });+}diff --git a/src/processors/src/export/vector-export.spec.ts b/src/processors/src/export/vector-export.spec.tsnew file mode 100644index 000000000..106fa6827--- /dev/null+++ b/src/processors/src/export/vector-export.spec.ts@@ -0,0 +1,224 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {convertArrowToSchema} from '@loaders.gl/schema-utils';+import {ALL_FIELD_TYPES, EXPORT_DATA_TYPE} from '@kepler.gl/constants';+import {formatCsv} from '@kepler.gl/reducers';+import {Field} from '@kepler.gl/types';+import {createDataContainer} from '@kepler.gl/utils';+import {tableFromIPC} from 'apache-arrow';++import {datasetToFeatureCollection, encodeDatasetExport} from './vector-export';+import {loadSqlJs} from './sql-js';++const fields = [+ {name: 'name', type: ALL_FIELD_TYPES.string},+ {name: 'secret', type: ALL_FIELD_TYPES.string},+ {name: 'lat', type: ALL_FIELD_TYPES.real},+ {name: 'lng', type: ALL_FIELD_TYPES.real}+] as Field[];++const rows = [+ ['Harbor', 'hidden-value', 37.8, -122.4],+ ['Pier', 'also-hidden', 37.81, -122.41]+];++const fieldPairs = [+ {+ defaultName: 'point',+ pair: {+ lat: {fieldIdx: 2, value: 'lat'},+ lng: {fieldIdx: 3, value: 'lng'}+ },+ suffix: ['lat', 'lng']+ }+];++function container() {+ return createDataContainer(rows);+}++describe('dataset export', () => {+ test('omits columns hidden in the attribute table', () => {+ const collection = datasetToFeatureCollection(container(), fields, {+ hiddenColumns: ['secret'],+ fieldPairs+ });++ expect(collection.features).toHaveLength(2);+ expect(collection.features[0].geometry).toEqual({+ type: 'Point',+ coordinates: [-122.4, 37.8]+ });+ expect(collection.features[0].properties).toEqual({+ name: 'Harbor',+ lat: 37.8,+ lng: -122.4+ });+ expect(collection.features[0].properties).not.toHaveProperty('secret');++ const csv = formatCsv(container(), fields, ['secret']);+ expect(csv).toContain('Harbor');+ expect(csv).not.toContain('hidden-value');+ expect(csv).not.toContain('secret');+ });++ test('writes geojson, kml, kmz, shapefile, geopackage, and geoparquet', async () => {+ const shared = {+ data: container(),+ fields,+ fieldPairs,+ hiddenColumns: ['secret'],+ layerName: 'places'+ };++ const geojson = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.GEOJSON});+ const parsed = JSON.parse(String(geojson.data));+ expect(parsed.features[1].properties.name).toBe('Pier');+ expect(parsed.features[1].properties.secret).toBeUndefined();+ expect(geojson.extension).toBe('geojson');++ const kml = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.KML});+ expect(String(kml.data)).toContain('<coordinates>-122.4,37.8</coordinates>');+ expect(String(kml.data)).not.toContain('hidden-value');++ const kmz = await encodeDatasetExport({...shared, dataType: EXPORT_DATA_TYPE.KMZ});+ expect(Buffer.from(kmz.data as Uint8Array).includes(Buffer.from('doc.kml'))).toBe(true);++ const shapefile = await encodeDatasetExport({+ ...shared,+ dataType: EXPORT_DATA_TYPE.SHAPEFILE+ });+ const shapefileBytes = Buffer.from(shapefile.data as Uint8Array);+ expect(shapefile.extension).toBe('zip');+ expect(shapefileBytes.includes(Buffer.from('places.shp'))).toBe(true);+ expect(shapefileBytes.includes(Buffer.from('places.dbf'))).toBe(true);+ expect(shapefileBytes.includes(Buffer.from('hidden-value'))).toBe(false);++ const geopackage = await encodeDatasetExport({+ ...shared,+ dataType: EXPORT_DATA_TYPE.GEOPACKAGE+ });+ const SQL = await loadSqlJs();+ const db = new SQL.Database(new Uint8Array(geopackage.data as Uint8Array));+ const exported = db.exec('SELECT name FROM places');+ expect(exported[0].values.map(row => row[0])).toEqual(['Harbor', 'Pier']);+ const columns = db.exec('PRAGMA table_info(places)');+ const columnNames = columns[0].values.map(row => row[1]);+ expect(columnNames).not.toContain('secret');+ db.close();++ const geoparquet = await encodeDatasetExport({+ ...shared,+ dataType: EXPORT_DATA_TYPE.GEOPARQUET+ });+ const parquetBytes = Buffer.from(geoparquet.data as Uint8Array);+ expect(parquetBytes.subarray(0, 4).toString()).toBe('PAR1');+ expect(parquetBytes.includes(Buffer.from('"encoding":"WKB"'))).toBe(true);+ expect(parquetBytes.includes(Buffer.from('hidden-value'))).toBe(false);+ });++ test('rejects mixed geometry families when exporting a shapefile', async () => {+ const geometryFields = [+ {name: '_geojson', type: ALL_FIELD_TYPES.geojson},+ {name: 'label', type: ALL_FIELD_TYPES.string}+ ] as Field[];+ const geometryRows = [+ [{type: 'Point', coordinates: [0, 0]}, 'point'],+ [+ {+ type: 'LineString',+ coordinates: [+ [0, 0],+ [1, 1]+ ]+ },+ 'line'+ ]+ ];++ await expect(+ encodeDatasetExport({+ data: createDataContainer(geometryRows),+ fields: geometryFields,+ dataType: EXPORT_DATA_TYPE.SHAPEFILE,+ layerName: 'mixed'+ })+ ).rejects.toThrow('Shapefile export requires all geometries to have the same shape family.');+ });++ test('writes geoparquet string columns the importer can read', async () => {+ const geoparquet = await encodeDatasetExport({+ data: container(),+ fields,+ fieldPairs,+ dataType: EXPORT_DATA_TYPE.GEOPARQUET,+ layerName: 'places'+ });+ const specifier = 'parquet-wasm/node';+ const imported = (await import(specifier)) as {+ readParquet?: (bytes: Uint8Array) => {intoIPCStream(): Uint8Array};+ default?: {readParquet: (bytes: Uint8Array) => {intoIPCStream(): Uint8Array}};+ };+ const api = imported.readParquet ? imported : imported.default;+ if (!api?.readParquet) {+ throw new Error('parquet-wasm failed to load');+ }+ const wasmTable = api.readParquet(new Uint8Array(geoparquet.data as Uint8Array));+ const arrowTable = tableFromIPC(wasmTable.intoIPCStream());++ expect(arrowTable.getChild('name')?.toArray()).toEqual(['Harbor', 'Pier']);+ expect(arrowTable.getChild('geometry')?.get(0)).toBeInstanceOf(Uint8Array);+ const schema = convertArrowToSchema(arrowTable.schema);+ expect(schema.fields.find(field => field.name === 'geometry')?.type).toBe('binary');+ expect(schema.fields.find(field => field.name === 'name')?.type).toBe('utf8');+ });++ test.each(['gpkg_contents', 'gpkg_geometry_columns', 'sqlite_sequence'])(+ 'sanitizes the reserved GeoPackage layer name %s',+ async layerName => {+ const geopackage = await encodeDatasetExport({+ data: container(),+ fields,+ fieldPairs,+ dataType: EXPORT_DATA_TYPE.GEOPACKAGE,+ layerName+ });+ const SQL = await loadSqlJs();+ const db = new SQL.Database(new Uint8Array(geopackage.data as Uint8Array));+ try {+ const contents = db.exec('SELECT table_name FROM gpkg_contents');+ const exportedTable = String(contents[0].values[0][0]);+ expect(exportedTable).not.toBe(layerName);+ expect(exportedTable).not.toMatch(/^sqlite_/i);+ expect(db.exec(`SELECT COUNT(*) FROM "${exportedTable}"`)[0].values[0][0]).toBe(2);+ } finally {+ db.close();+ }+ }+ );++ test('uses a geojson column as feature geometry and drops it from properties', async () => {+ const polygon = {+ type: 'Polygon',+ coordinates: [+ [+ [0, 0],+ [1, 0],+ [1, 1],+ [0, 1],+ [0, 0]+ ]+ ]+ };+ const geoFields = [+ {name: '_geojson', type: ALL_FIELD_TYPES.geojson},+ {name: 'label', type: ALL_FIELD_TYPES.string}+ ] as Field[];+ const geoRows = [[polygon, 'block']];+ const collection = datasetToFeatureCollection(createDataContainer(geoRows), geoFields);++ expect(collection.features[0].geometry).toEqual(polygon);+ expect(collection.features[0].properties).toEqual({label: 'block'});+ });+});diff --git a/src/processors/src/export/vector-export.ts b/src/processors/src/export/vector-export.tsnew file mode 100644index 000000000..87e2e0490--- /dev/null+++ b/src/processors/src/export/vector-export.ts@@ -0,0 +1,119 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {EXPORT_DATA_TYPE} from '@kepler.gl/constants';+import {Field, FieldPair} from '@kepler.gl/types';+import {DataContainerInterface} from '@kepler.gl/utils';+import {ZipWriter} from '@loaders.gl/zip';+import type {FeatureCollection} from 'geojson';++import {datasetToFeatureCollection} from './dataset-to-geojson';+import {writeGeoPackage} from './geopackage-writer';+import {writeGeoParquet} from './geoparquet-writer';+import {writeKml} from './kml-writer';+import {writeShapefile} from './shapefile-writer';++export {datasetToFeatureCollection} from './dataset-to-geojson';++export type EncodedDatasetExport = {+ data: BlobPart;+ extension: string;+ mimeType: string;+};++/** Turn a dataset label into a filesystem-safe export base name. */+export function sanitizeExportName(name: string): string {+ let sanitized = '';+ for (const char of name.trim()) {+ const code = char.charCodeAt(0);+ sanitized += code <= 0x1f || '<>:"/\\|?*'.includes(char) ? '-' : char;+ }+ sanitized = sanitized.replace(/\s+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');+ return sanitized || 'dataset';+}++function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {+ const copy = new Uint8Array(bytes.byteLength);+ copy.set(bytes);+ return copy.buffer;+}++async function zipEntries(entries: Record<string, Uint8Array>): Promise<Uint8Array> {+ const files = Object.fromEntries(+ Object.entries(entries).map(([name, bytes]) => [name, toArrayBuffer(bytes)])+ );+ return new Uint8Array(await ZipWriter.encode(files));+}++/**+ * Encode a dataset for download. Hidden attribute-table columns are omitted.+ * Geometry columns that are hidden are still used to build the feature geometry.+ */+export async function encodeDatasetExport(options: {+ data: DataContainerInterface;+ fields: Field[];+ fieldPairs?: FieldPair[];+ hiddenColumns?: string[];+ dataType: string;+ layerName: string;+}): Promise<EncodedDatasetExport> {+ const layerName = sanitizeExportName(options.layerName);+ const geojson = datasetToFeatureCollection(options.data, options.fields, {+ hiddenColumns: options.hiddenColumns,+ fieldPairs: options.fieldPairs+ });++ switch (options.dataType) {+ case EXPORT_DATA_TYPE.GEOJSON:+ return {+ data: JSON.stringify(geojson),+ extension: 'geojson',+ mimeType: 'application/geo+json'+ };+ case EXPORT_DATA_TYPE.KML:+ return {+ data: writeKml(geojson, layerName),+ extension: 'kml',+ mimeType: 'application/vnd.google-earth.kml+xml'+ };+ case EXPORT_DATA_TYPE.KMZ:+ return {+ data: await zipEntries({+ 'doc.kml': new TextEncoder().encode(writeKml(geojson, layerName))+ }),+ extension: 'kmz',+ mimeType: 'application/vnd.google-earth.kmz'+ };+ case EXPORT_DATA_TYPE.SHAPEFILE:+ return {+ data: await zipShapefile(geojson, layerName),+ extension: 'zip',+ mimeType: 'application/zip'+ };+ case EXPORT_DATA_TYPE.GEOPACKAGE:+ return {+ data: await writeGeoPackage(geojson, layerName),+ extension: 'gpkg',+ mimeType: 'application/geopackage+sqlite3'+ };+ case EXPORT_DATA_TYPE.GEOPARQUET:+ return {+ data: await writeGeoParquet(geojson),+ extension: 'parquet',+ mimeType: 'application/vnd.apache.parquet'+ };+ default:+ throw new Error(`Unsupported export format ${options.dataType}`);+ }+}++async function zipShapefile(geojson: FeatureCollection, baseName: string): Promise<Uint8Array> {+ const parts = writeShapefile(geojson);+ return zipEntries({+ [`${baseName}.shp`]: parts.shp,+ [`${baseName}.shx`]: parts.shx,+ [`${baseName}.dbf`]: parts.dbf,+ [`${baseName}.prj`]: parts.prj,+ [`${baseName}.cpg`]: parts.cpg+ });+}diff --git a/src/processors/src/index.ts b/src/processors/src/index.tsindex 2b24f7f64..c921aaa44 100644--- a/src/processors/src/index.ts+++ b/src/processors/src/index.ts@@ -8,3 +8,9 @@ export * from './types';export * from './kepler-csv-loader';export * from './loader-registry';export * from './shapefile-files';+export {+ datasetToFeatureCollection,+ encodeDatasetExport,+ sanitizeExportName+} from './export/vector-export';+export type {EncodedDatasetExport} from './export/vector-export';diff --git a/src/reducers/src/column-edit-updaters.spec.ts b/src/reducers/src/column-edit-updaters.spec.tsindex 0c22cb985..67c6e131c 100644--- a/src/reducers/src/column-edit-updaters.spec.ts+++ b/src/reducers/src/column-edit-updaters.spec.ts@@ -95,6 +95,7 @@ function withCities(): VisState {describe('column edits', () => {test('rename changes the field identity stored in layers, filters, and saved maps', () => {let state = withCities();+ state.datasets.cities.hiddenColumns = ['region', 'pop'];expect(state.layers).toHaveLength(1);expect(state.filters[0].name).toEqual(['region']);@@ -122,6 +123,7 @@ describe('column edits', () => {expect(state.charts[0].yAxis?.field?.name).toBe('pop');expect(state.charts[0].crossFilter?.fieldNames).toEqual({xAxis: 'area'});expect(state.interactionConfig.tooltip.config.fieldsToShow.cities[0].name).toBe('area');+ expect(state.datasets.cities.hiddenColumns).toEqual(['area', 'pop']);const saved = SchemaManager.save({visState: state,@@ -143,12 +145,14 @@ describe('column edits', () => {test('delete drops references and refuses the last column', () => {let state = withCities();+ state.datasets.cities.hiddenColumns = ['region', 'pop'];state = reduceVisState(state, VisStateActions.deleteTableColumn('cities', 'pop'));expect(state.datasets.cities.fields.map(field => field.name)).toEqual(['lat', 'lng', 'region']);expect(state.layers).toHaveLength(1);expect(state.filters).toHaveLength(1);expect(state.charts[0].yAxis?.field).toBeNull();+ expect(state.datasets.cities.hiddenColumns).toEqual(['region']);state = reduceVisState(state, VisStateActions.deleteTableColumn('cities', 'lat'));expect(state.layers).toHaveLength(0);diff --git a/src/reducers/src/export-utils.ts b/src/reducers/src/export-utils.tsindex f8a79eed3..60b70df41 100644--- a/src/reducers/src/export-utils.ts+++ b/src/reducers/src/export-utils.ts@@ -5,6 +5,7 @@ import {Blob} from 'global/window';import {csvFormatRows} from 'd3-dsv';import {EXPORT_DATA_TYPE} from '@kepler.gl/constants';+import {encodeDatasetExport, sanitizeExportName} from '@kepler.gl/processors';import {Field} from '@kepler.gl/types';import KeplerTable, {Datasets} from '@kepler.gl/table';@@ -21,7 +22,7 @@ interface StateType {appName?: string;}-export function exportData(state: StateType, options) {+export async function exportData(state: StateType, options): Promise<void> {const {visState, appName} = state;const {datasets} = visState;const {selectedDataset, dataType, filtered} = options;@@ -35,26 +36,48 @@ export function exportData(state: StateType, options) {return;}- selectedDatasets.forEach(selectedData => {- const {dataContainer, fields, label, filteredIdxCPU = []} = selectedData as KeplerTable;- const toExport = filtered- ? createIndexedDataContainer(dataContainer, filteredIdxCPU)- : dataContainer;+ const errors: unknown[] = [];+ for (const selectedData of selectedDatasets) {+ try {+ const {+ dataContainer,+ fields,+ label,+ filteredIdxCPU = [],+ hiddenColumns,+ fieldPairs+ } = selectedData as KeplerTable;+ const toExport = filtered+ ? createIndexedDataContainer(dataContainer, filteredIdxCPU)+ : dataContainer;+ const layerName = sanitizeExportName(label);- // start to export data according to selected data type- switch (dataType) {- case EXPORT_DATA_TYPE.CSV: {- const csv = formatCsv(toExport, fields);-- const fileBlob = new Blob([csv], {type: 'text/csv'});- downloadFile(fileBlob, `${filename}_${label}.csv`);- break;+ if (dataType === EXPORT_DATA_TYPE.CSV) {+ const csv = formatCsv(toExport, fields, hiddenColumns);+ downloadFile(new Blob([csv], {type: 'text/csv'}), `${filename}_${layerName}.csv`);+ continue;}- // TODO: support more file types.- default:- break;++ const file = await encodeDatasetExport({+ data: toExport,+ fields,+ fieldPairs,+ hiddenColumns,+ dataType,+ layerName+ });+ downloadFile(+ new Blob([file.data], {type: file.mimeType}),+ `${filename}_${layerName}.${file.extension}`+ );+ } catch (error) {+ errors.push(error);}- });+ }++ if (errors.length) {+ throw errors[0];+ }}/**@@ -63,13 +86,29 @@ export function exportData(state: StateType, options) {* @param fields `dataset.fields`* @returns csv string*/-export function formatCsv(data: DataContainerInterface, fields: Field[]): string {- const columns = fields.map(f => f.displayName || f.name);+export function formatCsv(+ data: DataContainerInterface,+ fields: Field[],+ hiddenColumns: string[] = []+): string {+ const hidden = new Set(hiddenColumns);+ const included = fields+ .map((field, index) => ({field, index}))+ .filter(({field}) => !hidden.has(field.name));+ const columns = included.map(({field}) => field.displayName || field.name);const formattedData = [columns];// parse geojson object as stringfor (const row of data.rows(true)) {- formattedData.push(row.map((d, i) => parseFieldValue(d, fields[i].type, fields[i])));+ formattedData.push(+ included.map(({field, index}) =>+ parseFieldValue(+ typeof row.valueAt === 'function' ? row.valueAt(index) : row[index],+ field.type,+ field+ )+ )+ );}return csvFormatRows(formattedData);diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.tsindex 62f856eaa..e298870b5 100644--- a/src/reducers/src/vis-state-updaters.ts+++ b/src/reducers/src/vis-state-updaters.ts@@ -196,6 +196,8 @@ import {copyTableAndUpdate,createNewDataEntry,pinTableColumns,+ showAllTableColumns,+ toggleHiddenTableColumn,setFilterGpuMode,sortDatasetByColumn} from '@kepler.gl/table';@@ -4377,11 +4379,14 @@ export function updateDatasetUpdater(newTable.filterRecordCPU = undefined;newTable.sortColumn = undefined;newTable.sortOrder = null;+ const newFieldsByName = new Set(newTable.fields.map(f => f.name));newTable.pinnedColumns = (existing.pinnedColumns || []).map(name => renames[name] ?? name)- .filter(name => newTable.getColumnFieldIdx(name) > -1);+ .filter(name => newFieldsByName.has(name));+ newTable.hiddenColumns = (existing.hiddenColumns || [])+ .map(name => renames[name] ?? name)+ .filter(name => newFieldsByName.has(name));- const newFieldsByName = new Set(newTable.fields.map(f => f.name));const newDatasets = {...state.datasets, [dataId]: newTable};// 2. reconcile filters@@ -5625,6 +5630,36 @@ export function pinTableColumnUpdater(return set(['datasets', dataId], newDataset, state);}+/**+ * @memberof visStateUpdaters+ * @public+ */+export function toggleTableColumnHiddenUpdater(+ state: VisState,+ {dataId, column}: VisStateActions.ToggleTableColumnHiddenUpdaterAction+): VisState {+ const dataset = state.datasets[dataId];+ if (!dataset) {+ return state;+ }+ return set(['datasets', dataId], toggleHiddenTableColumn(dataset, column), state);+}++/**+ * @memberof visStateUpdaters+ * @public+ */+export function showAllTableColumnsUpdater(+ state: VisState,+ {dataId}: VisStateActions.ShowAllTableColumnsUpdaterAction+): VisState {+ const dataset = state.datasets[dataId];+ if (!dataset) {+ return state;+ }+ return set(['datasets', dataId], showAllTableColumns(dataset), state);+}+/*** Copy column content as strings* @memberof visStateUpdatersdiff --git a/src/reducers/src/vis-state.ts b/src/reducers/src/vis-state.tsindex b839e5549..027e18fbf 100644--- a/src/reducers/src/vis-state.ts+++ b/src/reducers/src/vis-state.ts@@ -178,6 +178,10 @@ const actionHandler = {[ActionTypes.PIN_TABLE_COLUMN]: visStateUpdaters.pinTableColumnUpdater,+ [ActionTypes.TOGGLE_TABLE_COLUMN_HIDDEN]: visStateUpdaters.toggleTableColumnHiddenUpdater,++ [ActionTypes.SHOW_ALL_TABLE_COLUMNS]: visStateUpdaters.showAllTableColumnsUpdater,+[ActionTypes.COPY_TABLE_COLUMN]: visStateUpdaters.copyTableColumnUpdater,[ActionTypes.SET_COLUMN_DISPLAY_FORMAT]: visStateUpdaters.setColumnDisplayFormatUpdater,diff --git a/src/table/src/hidden-columns.spec.ts b/src/table/src/hidden-columns.spec.tsnew file mode 100644index 000000000..10d8fbca7--- /dev/null+++ b/src/table/src/hidden-columns.spec.ts@@ -0,0 +1,39 @@+// SPDX-License-Identifier: MIT+// Copyright contributors to the kepler.gl project++import {ALL_FIELD_TYPES} from '@kepler.gl/constants';++import KeplerTable, {showAllTableColumns, toggleHiddenTableColumn} from './kepler-table';++function makeTable() {+ const table = new KeplerTable({+ info: {id: 'places', label: 'places'},+ color: [0, 0, 0]+ });+ table.updateSchema({+ fields: [+ {name: 'name', type: ALL_FIELD_TYPES.string, analyzerType: ALL_FIELD_TYPES.string},+ {name: 'secret', type: ALL_FIELD_TYPES.string, analyzerType: ALL_FIELD_TYPES.string}+ ],+ rows: [['Harbor', 'hidden-value']]+ });+ return table;+}++describe('hidden table columns', () => {+ test('toggle hides a column and show all restores it', () => {+ const hidden = toggleHiddenTableColumn(makeTable(), 'secret');+ expect(hidden.hiddenColumns).toEqual(['secret']);++ const shown = toggleHiddenTableColumn(hidden, 'secret');+ expect(shown.hiddenColumns).toEqual([]);++ const both = toggleHiddenTableColumn(toggleHiddenTableColumn(makeTable(), 'secret'), 'name');+ expect(showAllTableColumns(both).hiddenColumns).toEqual([]);+ });++ test('ignores a column that is not on the dataset', () => {+ const table = makeTable();+ expect(toggleHiddenTableColumn(table, 'missing')).toBe(table);+ });+});diff --git a/src/table/src/index.ts b/src/table/src/index.tsindex 87e2aee87..1869a7be4 100644--- a/src/table/src/index.ts+++ b/src/table/src/index.ts@@ -7,6 +7,8 @@ export {findPointFieldPairs,copyTableAndUpdate,pinTableColumns,+ showAllTableColumns,+ toggleHiddenTableColumn,sortDatasetByColumn,copyTable,maybeToDatediff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.tsindex 351b79aa1..8295f918f 100644--- a/src/table/src/kepler-table.ts+++ b/src/table/src/kepler-table.ts@@ -231,6 +231,8 @@ class KeplerTable<F extends Field = Field> {sortOrder?: number[] | null;pinnedColumns?: string[];+ /** Attribute-table columns omitted from data export. */+ hiddenColumns?: string[];supportedFilterTypes?: string[] | null;disableDataOperation?: boolean;@@ -1013,6 +1015,30 @@ export function pinTableColumns(dataset: KeplerTable<Field>, column: string): Kereturn copyTableAndUpdate(dataset, {pinnedColumns});}+export function toggleHiddenTableColumn(+ dataset: KeplerTable<Field>,+ column: string+): KeplerTable<Field> {+ const field = dataset.fields.find(item => item.name === column);+ if (!field) {+ return dataset;+ }++ const hidden = dataset.hiddenColumns || [];+ const hiddenColumns = hidden.includes(field.name)+ ? hidden.filter(name => name !== field.name)+ : hidden.concat(field.name);++ return copyTableAndUpdate(dataset, {hiddenColumns});+}++export function showAllTableColumns(dataset: KeplerTable<Field>): KeplerTable<Field> {+ if (!dataset.hiddenColumns?.length) {+ return dataset;+ }+ return copyTableAndUpdate(dataset, {hiddenColumns: []});+}+export function copyTable(original: KeplerTable<Field>): KeplerTable<Field> {return Object.assign(Object.create(Object.getPrototypeOf(original)), original);}diff --git a/test/browser/components/modals/data-table-modal-test.js b/test/browser/components/modals/data-table-modal-test.jsindex 9c0479c06..a2e73ad3d 100644--- a/test/browser/components/modals/data-table-modal-test.js+++ b/test/browser/components/modals/data-table-modal-test.js@@ -225,23 +225,30 @@ test('Components -> DataTableModal.render: csv 1', t => {const expectedColMeta = {'gps_data.utc_timestamp': {name: 'gps_data.utc_timestamp',+ colIdx: 0,type: 'timestamp',format: 'YYYY-M-D H:m:s'},- 'gps_data.lat': {name: 'gps_data.lat', type: 'real'},- 'gps_data.lng': {name: 'gps_data.lng', type: 'real'},- 'gps_data.types': {name: 'gps_data.types', type: 'string'},- epoch: {name: 'epoch', type: 'timestamp', format: 'X'},- has_result: {name: 'has_result', type: 'boolean'},- uid: {name: 'uid', type: 'integer'},- time: {name: 'time', type: 'timestamp', format: 'YYYY-M-DTHH:mm:ss.SSSS'},- begintrip_ts_utc: {name: 'begintrip_ts_utc', type: 'timestamp', format: 'YYYY-M-D HH:mm:ssZZ'},+ 'gps_data.lat': {name: 'gps_data.lat', colIdx: 1, type: 'real'},+ 'gps_data.lng': {name: 'gps_data.lng', colIdx: 2, type: 'real'},+ 'gps_data.types': {name: 'gps_data.types', colIdx: 3, type: 'string'},+ epoch: {name: 'epoch', colIdx: 4, type: 'timestamp', format: 'X'},+ has_result: {name: 'has_result', colIdx: 5, type: 'boolean'},+ uid: {name: 'uid', colIdx: 6, type: 'integer'},+ time: {name: 'time', colIdx: 7, type: 'timestamp', format: 'YYYY-M-DTHH:mm:ss.SSSS'},+ begintrip_ts_utc: {+ name: 'begintrip_ts_utc',+ colIdx: 8,+ type: 'timestamp',+ format: 'YYYY-M-D HH:mm:ssZZ'+ },begintrip_ts_local: {name: 'begintrip_ts_local',+ colIdx: 9,type: 'timestamp',format: 'YYYY-M-D HH:mm:ssZZ'},- date: {name: 'date', type: 'date', format: 'YYYY-M-D'}+ date: {name: 'date', colIdx: 10, type: 'date', format: 'YYYY-M-D'}};t.deepEqual(props.columns, expectedColumns, 'DataTable should have the correct props.columns');diff --git a/test/node/reducers/ui-state-test.js b/test/node/reducers/ui-state-test.jsindex f8069fef8..fcced6624 100644--- a/test/node/reducers/ui-state-test.js+++ b/test/node/reducers/ui-state-test.js@@ -260,13 +260,13 @@ test('#uiStateReducer -> SET_EXPORT_SELECTED_DATASET', t => {});test('#uiStateReducer -> SET_EXPORT_DATA_TYPE', t => {- const newReducer = reducer(INITIAL_UI_STATE, setExportDataType(EXPORT_DATA_TYPE.JSON));+ const newReducer = reducer(INITIAL_UI_STATE, setExportDataType(EXPORT_DATA_TYPE.GEOJSON));const expectedState = {...INITIAL_UI_STATE,exportData: {...INITIAL_UI_STATE.exportData,- dataType: EXPORT_DATA_TYPE.JSON+ dataType: EXPORT_DATA_TYPE.GEOJSON}};diff --git a/test/node/reducers/vis-state-test.js b/test/node/reducers/vis-state-test.jsindex 68f96db1b..f5d05b23c 100644--- a/test/node/reducers/vis-state-test.js+++ b/test/node/reducers/vis-state-test.js@@ -6906,6 +6906,47 @@ test('#visStateReducer -> PIN_TABLE_COLUMN', t => {t.end();});+test('#visStateReducer -> TOGGLE_TABLE_COLUMN_HIDDEN and SHOW_ALL_TABLE_COLUMNS', t => {+ const initialState = CloneDeep(StateWFiles.visState);++ const hiddenState = reducer(+ initialState,+ VisStateActions.toggleTableColumnHidden(testCsvDataId, 'gps_data.lat')+ );+ assertDatasetIsTable(t, hiddenState.datasets[testCsvDataId]);+ t.deepEqual(+ hiddenState.datasets[testCsvDataId].hiddenColumns,+ ['gps_data.lat'],+ 'should hide a table column through the reducer'+ );++ const shownState = reducer(+ hiddenState,+ VisStateActions.toggleTableColumnHidden(testCsvDataId, 'gps_data.lat')+ );+ t.deepEqual(shownState.datasets[testCsvDataId].hiddenColumns, [], 'should show a hidden column');++ const allShownState = reducer(hiddenState, VisStateActions.showAllTableColumns(testCsvDataId));+ t.deepEqual(allShownState.datasets[testCsvDataId].hiddenColumns, [], 'should show all columns');++ const unknownColumnState = reducer(+ initialState,+ VisStateActions.toggleTableColumnHidden(testCsvDataId, 'missing')+ );+ t.equal(+ unknownColumnState.datasets[testCsvDataId].hiddenColumns,+ undefined,+ 'should ignore an unknown column'+ );+ t.equal(+ reducer(initialState, VisStateActions.toggleTableColumnHidden('missing', 'gps_data.lat')),+ initialState,+ 'should ignore an unknown dataset'+ );++ t.end();+});+test('#visStateReducer -> LOAD_FILES', async t => {drainTasksForTesting();diff --git a/yarn.lock b/yarn.lockindex 027153df0..e1390f8b2 100644--- a/yarn.lock+++ b/yarn.lock@@ -4354,8 +4354,11 @@ __metadata:"@loaders.gl/zip": "npm:^4.4.1""@mapbox/geojson-normalize": "npm:0.0.1""@turf/helpers": "npm:^7.2.0"+ "@types/sql.js": "npm:^1"apache-arrow: "npm:>=15.0.0"d3-dsv: "npm:^2.0.0"+ parquet-wasm: "npm:0.6.1"+ sql.js: "npm:1.13.0"type-analyzer: "npm:0.4.0"languageName: unknownlinkType: soft@@ -10311,6 +10314,13 @@ __metadata:languageName: nodelinkType: hard+"@types/emscripten@npm:*":+ version: 1.41.6+ resolution: "@types/emscripten@npm:1.41.6"+ checksum: 10c0/b51a730b6cb2848ac5e3f0536707bf656b6887be21f20e96b01e74a59ed52319dd0942f49b1072b1ca6d38df19e2d633f540df54d6ba1f7216be14af7373a41f+ languageName: node+ linkType: hard+"@types/estree-jsx@npm:^1.0.0":version: 1.0.5resolution: "@types/estree-jsx@npm:1.0.5"@@ -10922,6 +10932,16 @@ __metadata:languageName: nodelinkType: hard+"@types/sql.js@npm:^1":+ version: 1.4.11+ resolution: "@types/sql.js@npm:1.4.11"+ dependencies:+ "@types/emscripten": "npm:*"+ "@types/node": "npm:*"+ checksum: 10c0/eefb0408235cb645879103414b8ef0f13c2e7f2eac69c2a6c2838b72ec0be49985d6c6711b95bc8d79c208c7c5db95a51576a6ce7bfb49dd9edcc17721ffb40f+ languageName: node+ linkType: hard+"@types/stack-utils@npm:^2.0.0":version: 2.0.3resolution: "@types/stack-utils@npm:2.0.3"@@ -26343,7 +26363,7 @@ __metadata:languageName: nodelinkType: hard-"parquet-wasm@npm:^0.6.1":+"parquet-wasm@npm:0.6.1, parquet-wasm@npm:^0.6.1":version: 0.6.1resolution: "parquet-wasm@npm:0.6.1"checksum: 10c0/dcade4b006ff86edca711dacf3c71997b4a86985384fc183119f73d5dae1d0c94d3e7fa5b48845845df5db97d3eb445d6c137f3459a5d9a66a3b9b3bc0474497@@ -30543,6 +30563,13 @@ __metadata:languageName: nodelinkType: hard+"sql.js@npm:1.13.0":+ version: 1.13.0+ resolution: "sql.js@npm:1.13.0"+ checksum: 10c0/ecad62bf74082bf296a42db9bf001a79cd1b2fc954dbaddb97d689f7382c573ec827a692bd4490322fac47358d74c026258042466e258b34ea7c07b35d4a2e8e+ languageName: node+ linkType: hard+"ssf@npm:~0.11.2":version: 0.11.2resolution: "ssf@npm:0.11.2"