Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[fix] vector tile layer fixes (#2911)
Igor Dykhtaa6e151a0f947
diff --git a/examples/demo-app/src/actions.js b/examples/demo-app/src/actions.jsindex 3c05fd8a0..220d85c01 100644--- a/examples/demo-app/src/actions.js+++ b/examples/demo-app/src/actions.js@@ -36,12 +36,13 @@ export function initApp() {};}-export function loadRemoteResourceSuccess(response, config, options) {+export function loadRemoteResourceSuccess(response, config, options, remoteDatasetConfig) {return {type: LOAD_REMOTE_RESOURCE_SUCCESS,response,config,- options+ options,+ remoteDatasetConfig};}@@ -191,12 +192,17 @@ export function loadSample(options, pushRoute = true) {function loadRemoteSampleMap(options) {return dispatch => {// Load configuration first- const {configUrl, dataUrl} = options;+ const {configUrl, dataUrl, remoteDatasetConfigUrl} = options;+ const toLoad = [loadRemoteConfig(configUrl)];+ toLoad.push(dataUrl ? loadRemoteData(dataUrl) : null);+ // Load remote dataset config for tiled layers+ toLoad.push(remoteDatasetConfigUrl ? loadRemoteConfig(remoteDatasetConfigUrl) : null);- Promise.all([loadRemoteConfig(configUrl), loadRemoteData(dataUrl)]).then(- ([config, data]) => {+ Promise.all(toLoad).then(+ ([config, data, remoteDatasetConfig]) => {// TODO: these two actions can be merged- dispatch(loadRemoteResourceSuccess(data, config, options));+ dispatch(loadRemoteResourceSuccess(data, config, options, remoteDatasetConfig));+ // TODO: toggleModal when async dataset task is done, show the spinner until thendispatch(toggleModal(null));},error => {diff --git a/examples/demo-app/src/app.tsx b/examples/demo-app/src/app.tsxindex 66ca90334..f19f98f46 100644--- a/examples/demo-app/src/app.tsx+++ b/examples/demo-app/src/app.tsx@@ -219,7 +219,7 @@ const App = props => {label: 'Railroads',id: 'railroads.pmtiles',color: [255, 0, 0],- type: 'vectorTile'+ type: 'vector-tile'},data: {rows: [],@@ -235,7 +235,8 @@ const App = props => {metadata: {name: 'output.pmtiles',description: 'output.pmtiles',- type: 'pmtiles',+ type: 'remote',+ remoteTileFormat: 'pmtiles',tilesetDataUrl:'https://4sq-studio-public.s3.us-west-2.amazonaws.com/pmtiles-test/161727fe-7952-4e57-aa05-850b3086b0b2.pmtiles',tilesetMetadataUrl:diff --git a/examples/demo-app/src/components/map-control/map-control.js b/examples/demo-app/src/components/map-control/map-control.jsindex 27ee2e32e..a512e3035 100644--- a/examples/demo-app/src/components/map-control/map-control.js+++ b/examples/demo-app/src/components/map-control/map-control.js@@ -125,7 +125,9 @@ export function SampleMapPanel(props) {<div className="project-links"><LinkButtonlabel="Data"- href={getURL(props.currentSample.dataUrl)}+ href={getURL(+ props.currentSample.dataUrl || props.currentSample.remoteDatasetConfigUrl+ )}iconComponent={Icons.Files}height="15px"/>diff --git a/examples/demo-app/src/reducers/index.js b/examples/demo-app/src/reducers/index.jsindex 24c174ed2..72774e29e 100644--- a/examples/demo-app/src/reducers/index.js+++ b/examples/demo-app/src/reducers/index.js@@ -93,26 +93,32 @@ export const loadRemoteResourceSuccess = (state, action) => {const datasetId = action.options.id || generateHashId(6);const {dataUrl} = action.options;- const {shape} = action.response;+ const {shape} = dataUrl ? action.response : {};let processorMethod = processRowObject;let unprocessedData = action.response;- if (shape === 'arrow-table') {- processorMethod = processArrowTable;- } else if (shape === 'object-row-table') {- processorMethod = processRowObject;- unprocessedData = action.response.data;- } else if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {- processorMethod = processGeojson;- } else {- throw new Error('Failed to select data processor');++ if (dataUrl) {+ if (shape === 'arrow-table') {+ processorMethod = processArrowTable;+ } else if (shape === 'object-row-table') {+ processorMethod = processRowObject;+ unprocessedData = action.response.data;+ } else if (dataUrl.includes('.json') || dataUrl.includes('.geojson')) {+ processorMethod = processGeojson;+ } else {+ throw new Error('Failed to select data processor');+ }}- const datasets = {- info: {- id: datasetId- },- data: processorMethod(unprocessedData)- };+ const datasets = dataUrl+ ? {+ info: {+ id: datasetId+ },+ data: processorMethod(unprocessedData)+ }+ : // remote datasets like vector tile datasets+ action.remoteDatasetConfig;const config = action.config ? KeplerGlSchema.parseSavedConfig(action.config) : null;diff --git a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts b/src/components/src/hooks/use-fetch-vector-tile-metadata.tsindex ce671b7fc..b996c506c 100644--- a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts+++ b/src/components/src/hooks/use-fetch-vector-tile-metadata.ts@@ -6,12 +6,12 @@ import {useCallback, useEffect, useState} from 'react';import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';-import {VectorTileType} from '@kepler.gl/constants';+import {RemoteTileFormat} from '@kepler.gl/constants';import {getMVTMetadata, VectorTileMetadata} from '@kepler.gl/table';type FetchVectorTileMetadataProps = {url: string | null;- type: VectorTileType;+ remoteTileFormat: RemoteTileFormat;process?: (json: PMTilesMetadata | TileJSON) => VectorTileMetadata | Error | null;};@@ -35,7 +35,7 @@ type FetchVectorTileMetadataReturn = {/** Hook to fetch and return mvt or pmtiles metadata. */export default function useFetchVectorTileMetadata({- type,+ remoteTileFormat,url,process = DEFAULT_PROCESS_FUNCTION}: FetchVectorTileMetadataProps): FetchVectorTileMetadataReturn {@@ -67,7 +67,7 @@ export default function useFetchVectorTileMetadata({try {let metadata: PMTilesMetadata | TileJSON | null = null;- if (type === VectorTileType.MVT) {+ if (remoteTileFormat === RemoteTileFormat.MVT) {metadata = await getMVTMetadata(url);// MVTSource returns messy partial metadata@@ -94,7 +94,7 @@ export default function useFetchVectorTileMetadata({};getAndProcessMetadata();- }, [url, type, setProcessedData]);+ }, [url, remoteTileFormat, setProcessedData]);return {data, loading, error};}diff --git a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsxindex 56a291b21..3ec5db7b2 100644--- a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx+++ b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx@@ -4,10 +4,16 @@import React, {useCallback, useEffect, useState, useMemo} from 'react';import styled from 'styled-components';-import {DatasetType, VectorTileType} from '@kepler.gl/constants';-import {getMetaUrl, parseVectorMetadata, VectorTileMetadata} from '@kepler.gl/table';+import {+ DatasetType,+ RemoteTileFormat,+ VectorTileDatasetMetadata,+ REMOTE_TILE+} from '@kepler.gl/constants';import {TileJSON} from '@loaders.gl/mvt';import {PMTilesMetadata} from '@loaders.gl/pmtiles';+import {getMetaUrl, parseVectorMetadata, VectorTileMetadata} from '@kepler.gl/table';+import {Merge} from '@kepler.gl/types';import {default as useFetchVectorTileMetadata} from '../../hooks/use-fetch-vector-tile-metadata';import {DatasetCreationAttributes, MetaResponse} from './common';@@ -34,16 +40,24 @@ export type VectorTilesetFormData = {const isPMTilesUrl = (url?: string | null) => url?.includes('.pmtiles');+export type VectorTileDatasetCreationAttributes = Merge<+ DatasetCreationAttributes,+ {+ metadata: VectorTileDatasetMetadata;+ }+>;+export function getDatasetAttributesFromVectorTile({name,dataUrl,metadataUrl-}: VectorTilesetFormData): DatasetCreationAttributes {+}: VectorTilesetFormData): VectorTileDatasetCreationAttributes {return {name,type: DatasetType.VECTOR_TILE,metadata: {- type: isPMTilesUrl(dataUrl) ? VectorTileType.PMTILES : VectorTileType.MVT,+ type: REMOTE_TILE,+ remoteTileFormat: isPMTilesUrl(dataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,tilesetDataUrl: dataUrl,tilesetMetadataUrl: metadataUrl}@@ -114,7 +128,7 @@ const TilesetVectorForm: React.FC<TilesetVectorFormProps> = ({setResponse}) => {error: metaError} = useFetchVectorTileMetadata({url: metadataUrl,- type: isPMTilesUrl(metadataUrl) ? VectorTileType.PMTILES : VectorTileType.MVT,+ remoteTileFormat: isPMTilesUrl(metadataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,process});diff --git a/src/constants/src/dataset.ts b/src/constants/src/dataset.tsindex 4af6da352..3d11bf8c1 100644--- a/src/constants/src/dataset.ts+++ b/src/constants/src/dataset.ts@@ -3,20 +3,19 @@export enum DatasetType {LOCAL = 'local',- VECTOR_TILE = 'vectorTile'+ VECTOR_TILE = 'vector-tile'}-export enum TileType {- VECTOR_TILE = 'vectorTile'-}--export enum VectorTileType {+export enum RemoteTileFormat {MVT = 'mvt',PMTILES = 'pmtiles'}+export const REMOTE_TILE = 'remote';+export type VectorTileDatasetMetadata = {- type: VectorTileType;+ type: typeof REMOTE_TILE;+ remoteTileFormat: RemoteTileFormat;tilesetDataUrl: string;tilesetMetadataUrl?: string;};diff --git a/src/layers/src/index.ts b/src/layers/src/index.tsindex e25ff6fe4..1f89ea8e4 100644--- a/src/layers/src/index.ts+++ b/src/layers/src/index.ts@@ -40,7 +40,7 @@ import {default as S2GeometryLayer} from './s2-geometry-layer/s2-geometry-layer'export {defaultElevation as s2DefaultElevation} from './s2-geometry-layer/s2-geometry-layer';export {getS2Center} from './s2-geometry-layer/s2-utils';export {default as AggregationLayer} from './aggregation-layer';-//import {default as VectorTileLayer} from './vector-tile/vector-tile-layer';+// import {default as VectorTileLayer} from './vector-tile/vector-tile-layer';export {default as VectorTileIcon} from './vector-tile/vector-tile-icon';export {default as VectorTileLayer} from './vector-tile/vector-tile-layer';diff --git a/src/layers/src/vector-tile/vector-tile-layer.ts b/src/layers/src/vector-tile/vector-tile-layer.tsindex def81db24..ba5eb9470 100644--- a/src/layers/src/vector-tile/vector-tile-layer.ts+++ b/src/layers/src/vector-tile/vector-tile-layer.ts@@ -13,8 +13,8 @@ import GL from '@luma.gl/constants';import {notNullorUndefined} from '@kepler.gl/common-utils';import {DatasetType,- TileType,- VectorTileType,+ LAYER_TYPES,+ RemoteTileFormat,VectorTileDatasetMetadata,SCALE_TYPES,CHANNEL_SCALES,@@ -210,7 +210,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu}get type() {- return TileType.VECTOR_TILE;+ return LAYER_TYPES.vectorTile;}get name(): string {@@ -406,7 +406,8 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featuif (dataset?.type === DatasetType.VECTOR_TILE) {const datasetMetadata = dataset.metadata as VectorTileMetadata & VectorTileDatasetMetadata;- if (datasetMetadata?.type === VectorTileType.MVT) {+ const remoteTileFormat = datasetMetadata?.remoteTileFormat;+ if (remoteTileFormat === RemoteTileFormat.MVT) {const transformFetch = async (input: RequestInfo | URL, init?: RequestInit | undefined) => {const requestData: RequestParameters = {url: input as string,@@ -428,7 +429,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu}}): null;- } else if (datasetMetadata?.type === VectorTileType.PMTILES) {+ } else if (remoteTileFormat === RemoteTileFormat.PMTILES) {// TODO: to render image pmtiles need to use TileLayer and BitmapLayer (https://github.com/visgl/loaders.gl/blob/master/examples/website/tiles/components/tile-source-layer.ts)tilesetDataUrl = datasetMetadata?.tilesetDataUrl;tileSource = tilesetDataUrl ? PMTilesSource.createDataSource(tilesetDataUrl, {}) : null;diff --git a/src/table/src/dataset-utils.ts b/src/table/src/dataset-utils.tsindex e7a20188a..fc2456bb9 100644--- a/src/table/src/dataset-utils.ts+++ b/src/table/src/dataset-utils.ts@@ -6,7 +6,7 @@ import KeplerTable, {Datasets} from './kepler-table';import {ProtoDataset, RGBColor} from '@kepler.gl/types';import Task from 'react-palm/tasks';-import {DatasetType, VectorTileType} from '@kepler.gl/constants';+import {DatasetType, RemoteTileFormat, VectorTileDatasetMetadata} from '@kepler.gl/constants';import {hexToRgb,validateInputData,@@ -95,12 +95,16 @@ type CreateTableProps = {data: any;};-async function createTable(dataasetInfo: CreateTableProps) {- const {info, color, opts, data} = dataasetInfo;+async function createTable(datasetInfo: CreateTableProps) {+ const {info, color, opts, data} = datasetInfo;// update metadata for remote tiled datasets- const refreshedMetadata = await refreshMetadata(dataasetInfo);- const metadata = refreshedMetadata ? {...opts.metadata, ...refreshedMetadata} : opts.metadata;+ const refreshedMetadata = await refreshRemoteData(datasetInfo);+ let metadata = opts.metadata;+ if (refreshedMetadata) {+ metadata = {...opts.metadata, ...refreshedMetadata};+ data.fields = metadata?.fields;+ }const TableClass = getApplicationConfig().table ?? KeplerTable;const table = new TableClass({@@ -121,16 +125,17 @@ const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');* @param datasetInfo* @returns*/-async function refreshMetadata(datasetInfo: CreateTableProps) {+async function refreshRemoteData(datasetInfo: CreateTableProps) {// so far only vector tile layers should refresh metadataif (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {return null;}- const {type, tilesetMetadataUrl} = datasetInfo.opts.metadata || {};+ const {remoteTileFormat, tilesetMetadataUrl} =+ (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};if (- !(type === VectorTileType.PMTILES || type === VectorTileType.MVT) ||+ !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||typeof tilesetMetadataUrl !== 'string') {return null;@@ -138,7 +143,7 @@ async function refreshMetadata(datasetInfo: CreateTableProps) {try {let rawMetadata: PMTilesMetadata | TileJSON | null = null;- if (type === VectorTileType.MVT) {+ if (remoteTileFormat === RemoteTileFormat.MVT) {rawMetadata = await getMVTMetadata(tilesetMetadataUrl);} else {const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});