Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[fix] vector tile layer fixes (#2911)

Igor Dykhtaa6e151a0f947
diff --git a/examples/demo-app/src/actions.js b/examples/demo-app/src/actions.js
index 3c05fd8a0..220d85c01 100644
--- a/examples/demo-app/src/actions.js
+++ b/examples/demo-app/src/actions.js
@@ -36,12 +36,13 @@ export function initApp() {
};
}
-export function loadRemoteResourceSuccess(response, config, options) {
+export function loadRemoteResourceSuccess(response, config, options, remoteDatasetConfig) {
return {
type: LOAD_REMOTE_RESOURCE_SUCCESS,
response,
config,
- options
+ options,
+ remoteDatasetConfig
};
}
@@ -191,12 +192,17 @@ export function loadSample(options, pushRoute = true) {
function loadRemoteSampleMap(options) {
return dispatch => {
// Load configuration first
- const {configUrl, dataUrl} = options;
+ const {configUrl, dataUrl, remoteDatasetConfigUrl} = options;
+ const toLoad = [loadRemoteConfig(configUrl)];
+ toLoad.push(dataUrl ? loadRemoteData(dataUrl) : null);
+ // Load remote dataset config for tiled layers
+ toLoad.push(remoteDatasetConfigUrl ? loadRemoteConfig(remoteDatasetConfigUrl) : null);
- Promise.all([loadRemoteConfig(configUrl), loadRemoteData(dataUrl)]).then(
- ([config, data]) => {
+ Promise.all(toLoad).then(
+ ([config, data, remoteDatasetConfig]) => {
// TODO: these two actions can be merged
- dispatch(loadRemoteResourceSuccess(data, config, options));
+ dispatch(loadRemoteResourceSuccess(data, config, options, remoteDatasetConfig));
+ // TODO: toggleModal when async dataset task is done, show the spinner until then
dispatch(toggleModal(null));
},
error => {
diff --git a/examples/demo-app/src/app.tsx b/examples/demo-app/src/app.tsx
index 66ca90334..f19f98f46 100644
--- a/examples/demo-app/src/app.tsx
+++ b/examples/demo-app/src/app.tsx
@@ -219,7 +219,7 @@ const App = props => {
label: 'Railroads',
id: 'railroads.pmtiles',
color: [255, 0, 0],
- type: 'vectorTile'
+ type: 'vector-tile'
},
data: {
rows: [],
@@ -235,7 +235,8 @@ const App = props => {
metadata: {
name: 'output.pmtiles',
description: 'output.pmtiles',
- type: 'pmtiles',
+ type: 'remote',
+ remoteTileFormat: 'pmtiles',
tilesetDataUrl:
'https://4sq-studio-public.s3.us-west-2.amazonaws.com/pmtiles-test/161727fe-7952-4e57-aa05-850b3086b0b2.pmtiles',
tilesetMetadataUrl:
diff --git a/examples/demo-app/src/components/map-control/map-control.js b/examples/demo-app/src/components/map-control/map-control.js
index 27ee2e32e..a512e3035 100644
--- a/examples/demo-app/src/components/map-control/map-control.js
+++ b/examples/demo-app/src/components/map-control/map-control.js
@@ -125,7 +125,9 @@ export function SampleMapPanel(props) {
<div className="project-links">
<LinkButton
label="Data"
- href={getURL(props.currentSample.dataUrl)}
+ href={getURL(
+ props.currentSample.dataUrl || props.currentSample.remoteDatasetConfigUrl
+ )}
iconComponent={Icons.Files}
height="15px"
/>
diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.js
index 24c174ed2..72774e29e 100644
--- a/examples/demo-app/src/reducers/index.js
+++ b/examples/demo-app/src/reducers/index.js
@@ -93,26 +93,32 @@ export const loadRemoteResourceSuccess = (state, action) => {
const datasetId = action.options.id || generateHashId(6);
const {dataUrl} = action.options;
- const {shape} = action.response;
+ const {shape} = dataUrl ? action.response : {};
let processorMethod = processRowObject;
let unprocessedData = action.response;
- if (shape === 'arrow-table') {
- processorMethod = processArrowTable;
- } else if (shape === 'object-row-table') {
- processorMethod = processRowObject;
- unprocessedData = action.response.data;
- } else if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {
- processorMethod = processGeojson;
- } else {
- throw new Error('Failed to select data processor');
+
+ if (dataUrl) {
+ if (shape === 'arrow-table') {
+ processorMethod = processArrowTable;
+ } else if (shape === 'object-row-table') {
+ processorMethod = processRowObject;
+ unprocessedData = action.response.data;
+ } else if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {
+ processorMethod = processGeojson;
+ } else {
+ throw new Error('Failed to select data processor');
+ }
}
- const datasets = {
- info: {
- id: datasetId
- },
- data: processorMethod(unprocessedData)
- };
+ const datasets = dataUrl
+ ? {
+ info: {
+ id: datasetId
+ },
+ data: processorMethod(unprocessedData)
+ }
+ : // remote datasets like vector tile datasets
+ action.remoteDatasetConfig;
const config = action.config ? KeplerGlSchema.parseSavedConfig(action.config) : null;
diff --git a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts b/src/components/src/hooks/use-fetch-vector-tile-metadata.ts
index ce671b7fc..b996c506c 100644
--- a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts
+++ b/src/components/src/hooks/use-fetch-vector-tile-metadata.ts
@@ -6,12 +6,12 @@ import {useCallback, useEffect, useState} from 'react';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
-import {VectorTileType} from '@kepler.gl/constants';
+import {RemoteTileFormat} from '@kepler.gl/constants';
import {getMVTMetadata, VectorTileMetadata} from '@kepler.gl/table';
type FetchVectorTileMetadataProps = {
url: string | null;
- type: VectorTileType;
+ remoteTileFormat: RemoteTileFormat;
process?: (json: PMTilesMetadata | TileJSON) => VectorTileMetadata | Error | null;
};
@@ -35,7 +35,7 @@ type FetchVectorTileMetadataReturn = {
/** Hook to fetch and return mvt or pmtiles metadata. */
export default function useFetchVectorTileMetadata({
- type,
+ remoteTileFormat,
url,
process = DEFAULT_PROCESS_FUNCTION
}: FetchVectorTileMetadataProps): FetchVectorTileMetadataReturn {
@@ -67,7 +67,7 @@ export default function useFetchVectorTileMetadata({
try {
let metadata: PMTilesMetadata | TileJSON | null = null;
- if (type === VectorTileType.MVT) {
+ if (remoteTileFormat === RemoteTileFormat.MVT) {
metadata = await getMVTMetadata(url);
// MVTSource returns messy partial metadata
@@ -94,7 +94,7 @@ export default function useFetchVectorTileMetadata({
};
getAndProcessMetadata();
- }, [url, type, setProcessedData]);
+ }, [url, remoteTileFormat, setProcessedData]);
return {data, loading, error};
}
diff --git a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx
index 56a291b21..3ec5db7b2 100644
--- a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx
+++ b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx
@@ -4,10 +4,16 @@
import React, {useCallback, useEffect, useState, useMemo} from 'react';
import styled from 'styled-components';
-import {DatasetType, VectorTileType} from '@kepler.gl/constants';
-import {getMetaUrl, parseVectorMetadata, VectorTileMetadata} from '@kepler.gl/table';
+import {
+ DatasetType,
+ RemoteTileFormat,
+ VectorTileDatasetMetadata,
+ REMOTE_TILE
+} from '@kepler.gl/constants';
import {TileJSON} from '@loaders.gl/mvt';
import {PMTilesMetadata} from '@loaders.gl/pmtiles';
+import {getMetaUrl, parseVectorMetadata, VectorTileMetadata} from '@kepler.gl/table';
+import {Merge} from '@kepler.gl/types';
import {default as useFetchVectorTileMetadata} from '../../hooks/use-fetch-vector-tile-metadata';
import {DatasetCreationAttributes, MetaResponse} from './common';
@@ -34,16 +40,24 @@ export type VectorTilesetFormData = {
const isPMTilesUrl = (url?: string | null) => url?.includes('.pmtiles');
+export type VectorTileDatasetCreationAttributes = Merge<
+ DatasetCreationAttributes,
+ {
+ metadata: VectorTileDatasetMetadata;
+ }
+>;
+
export function getDatasetAttributesFromVectorTile({
name,
dataUrl,
metadataUrl
-}: VectorTilesetFormData): DatasetCreationAttributes {
+}: VectorTilesetFormData): VectorTileDatasetCreationAttributes {
return {
name,
type: DatasetType.VECTOR_TILE,
metadata: {
- type: isPMTilesUrl(dataUrl) ? VectorTileType.PMTILES : VectorTileType.MVT,
+ type: REMOTE_TILE,
+ remoteTileFormat: isPMTilesUrl(dataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,
tilesetDataUrl: dataUrl,
tilesetMetadataUrl: metadataUrl
}
@@ -114,7 +128,7 @@ const TilesetVectorForm: React.FC<TilesetVectorFormProps> = ({setResponse}) => {
error: metaError
} = useFetchVectorTileMetadata({
url: metadataUrl,
- type: isPMTilesUrl(metadataUrl) ? VectorTileType.PMTILES : VectorTileType.MVT,
+ remoteTileFormat: isPMTilesUrl(metadataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,
process
});
diff --git a/src/constants/src/dataset.ts b/src/constants/src/dataset.ts
index 4af6da352..3d11bf8c1 100644
--- a/src/constants/src/dataset.ts
+++ b/src/constants/src/dataset.ts
@@ -3,20 +3,19 @@
export enum DatasetType {
LOCAL = 'local',
- VECTOR_TILE = 'vectorTile'
+ VECTOR_TILE = 'vector-tile'
}
-export enum TileType {
- VECTOR_TILE = 'vectorTile'
-}
-
-export enum VectorTileType {
+export enum RemoteTileFormat {
MVT = 'mvt',
PMTILES = 'pmtiles'
}
+export const REMOTE_TILE = 'remote';
+
export type VectorTileDatasetMetadata = {
- type: VectorTileType;
+ type: typeof REMOTE_TILE;
+ remoteTileFormat: RemoteTileFormat;
tilesetDataUrl: string;
tilesetMetadataUrl?: string;
};
diff --git a/src/layers/src/index.ts b/src/layers/src/index.ts
index e25ff6fe4..1f89ea8e4 100644
--- a/src/layers/src/index.ts
+++ b/src/layers/src/index.ts
@@ -40,7 +40,7 @@ import {default as S2GeometryLayer} from './s2-geometry-layer/s2-geometry-layer'
export {defaultElevation as s2DefaultElevation} from './s2-geometry-layer/s2-geometry-layer';
export {getS2Center} from './s2-geometry-layer/s2-utils';
export {default as AggregationLayer} from './aggregation-layer';
-//import {default as VectorTileLayer} from './vector-tile/vector-tile-layer';
+// import {default as VectorTileLayer} from './vector-tile/vector-tile-layer';
export {default as VectorTileIcon} from './vector-tile/vector-tile-icon';
export {default as VectorTileLayer} from './vector-tile/vector-tile-layer';
diff --git a/src/layers/src/vector-tile/vector-tile-layer.ts b/src/layers/src/vector-tile/vector-tile-layer.ts
index def81db24..ba5eb9470 100644
--- a/src/layers/src/vector-tile/vector-tile-layer.ts
+++ b/src/layers/src/vector-tile/vector-tile-layer.ts
@@ -13,8 +13,8 @@ import GL from '@luma.gl/constants';
import {notNullorUndefined} from '@kepler.gl/common-utils';
import {
DatasetType,
- TileType,
- VectorTileType,
+ LAYER_TYPES,
+ RemoteTileFormat,
VectorTileDatasetMetadata,
SCALE_TYPES,
CHANNEL_SCALES,
@@ -210,7 +210,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
}
get type() {
- return TileType.VECTOR_TILE;
+ return LAYER_TYPES.vectorTile;
}
get name(): string {
@@ -406,7 +406,8 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
if (dataset?.type === DatasetType.VECTOR_TILE) {
const datasetMetadata = dataset.metadata as VectorTileMetadata & VectorTileDatasetMetadata;
- if (datasetMetadata?.type === VectorTileType.MVT) {
+ const remoteTileFormat = datasetMetadata?.remoteTileFormat;
+ if (remoteTileFormat === RemoteTileFormat.MVT) {
const transformFetch = async (input: RequestInfo | URL, init?: RequestInit | undefined) => {
const requestData: RequestParameters = {
url: input as string,
@@ -428,7 +429,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
}
})
: null;
- } else if (datasetMetadata?.type === VectorTileType.PMTILES) {
+ } else if (remoteTileFormat === RemoteTileFormat.PMTILES) {
// TODO: to render image pmtiles need to use TileLayer and BitmapLayer (https://github.com/visgl/loaders.gl/blob/master/examples/website/tiles/components/tile-source-layer.ts)
tilesetDataUrl = datasetMetadata?.tilesetDataUrl;
tileSource = tilesetDataUrl ? PMTilesSource.createDataSource(tilesetDataUrl, {}) : null;
diff --git a/src/table/src/dataset-utils.ts b/src/table/src/dataset-utils.ts
index e7a20188a..fc2456bb9 100644
--- a/src/table/src/dataset-utils.ts
+++ b/src/table/src/dataset-utils.ts
@@ -6,7 +6,7 @@ import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';
-import {DatasetType, VectorTileType} from '@kepler.gl/constants';
+import {DatasetType, RemoteTileFormat, VectorTileDatasetMetadata} from '@kepler.gl/constants';
import {
hexToRgb,
validateInputData,
@@ -95,12 +95,16 @@ type CreateTableProps = {
data: any;
};
-async function createTable(dataasetInfo: CreateTableProps) {
- const {info, color, opts, data} = dataasetInfo;
+async function createTable(datasetInfo: CreateTableProps) {
+ const {info, color, opts, data} = datasetInfo;
// update metadata for remote tiled datasets
- const refreshedMetadata = await refreshMetadata(dataasetInfo);
- const metadata = refreshedMetadata ? {...opts.metadata, ...refreshedMetadata} : opts.metadata;
+ const refreshedMetadata = await refreshRemoteData(datasetInfo);
+ let metadata = opts.metadata;
+ if (refreshedMetadata) {
+ metadata = {...opts.metadata, ...refreshedMetadata};
+ data.fields = metadata?.fields;
+ }
const TableClass = getApplicationConfig().table ?? KeplerTable;
const table = new TableClass({
@@ -121,16 +125,17 @@ const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');
* @param datasetInfo
* @returns
*/
-async function refreshMetadata(datasetInfo: CreateTableProps) {
+async function refreshRemoteData(datasetInfo: CreateTableProps) {
// so far only vector tile layers should refresh metadata
if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {
return null;
}
- const {type, tilesetMetadataUrl} = datasetInfo.opts.metadata || {};
+ const {remoteTileFormat, tilesetMetadataUrl} =
+ (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};
if (
- !(type === VectorTileType.PMTILES || type === VectorTileType.MVT) ||
+ !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
typeof tilesetMetadataUrl !== 'string'
) {
return null;
@@ -138,7 +143,7 @@ async function refreshMetadata(datasetInfo: CreateTableProps) {
try {
let rawMetadata: PMTilesMetadata | TileJSON | null = null;
- if (type === VectorTileType.MVT) {
+ if (remoteTileFormat === RemoteTileFormat.MVT) {
rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
} else {
const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});