Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.
[fix] fixes for vector-tile layer (#3013)
Igor Dykhta7107e41776fb
diff --git a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts b/src/components/src/hooks/use-fetch-vector-tile-metadata.tsindex b996c506c..6c59042d4 100644--- a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts+++ b/src/components/src/hooks/use-fetch-vector-tile-metadata.ts@@ -1,16 +1,17 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project-import {useCallback, useEffect, useState} from 'react';+import {useEffect, useState} from 'react';-import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';+import {TileJSON} from '@loaders.gl/mvt';import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';import {RemoteTileFormat} from '@kepler.gl/constants';-import {getMVTMetadata, VectorTileMetadata} from '@kepler.gl/table';+import {getMVTMetadata, VectorTileMetadata, getFieldsFromTile} from '@kepler.gl/table';type FetchVectorTileMetadataProps = {- url: string | null;+ metadataUrl: string | null;+ tilesetUrl: string | null;remoteTileFormat: RemoteTileFormat;process?: (json: PMTilesMetadata | TileJSON) => VectorTileMetadata | Error | null;};@@ -36,39 +37,25 @@ type FetchVectorTileMetadataReturn = {/** Hook to fetch and return mvt or pmtiles metadata. */export default function useFetchVectorTileMetadata({remoteTileFormat,- url,+ tilesetUrl,+ metadataUrl,process = DEFAULT_PROCESS_FUNCTION}: FetchVectorTileMetadataProps): FetchVectorTileMetadataReturn {const [error, setError] = useState<Error | null>(null);const [data, setData] = useState<VectorTileMetadata | null>(null);const [loading, setLoading] = useState<boolean>(false);- const setProcessedData = useCallback(- (value: PMTilesMetadata | TileJSON | null) => {- if (!value) {- return setError(value);- }- const processedData = process(value);- if (processedData instanceof Error) {- setError(processedData);- } else {- setData(processedData);- }- },- [setError, setData, process]- );-useEffect(() => {const getAndProcessMetadata = async () => {setError(null);setData(null);- if (url) {+ if (metadataUrl) {setLoading(true);try {let metadata: PMTilesMetadata | TileJSON | null = null;if (remoteTileFormat === RemoteTileFormat.MVT) {- metadata = await getMVTMetadata(url);+ metadata = await getMVTMetadata(metadataUrl);// MVTSource returns messy partial metadata// MVTSource.createDataSource('', {@@ -77,7 +64,7 @@ export default function useFetchVectorTileMetadata({// }// })} else {- const tileSource = PMTilesSource.createDataSource(url, {});+ const tileSource = PMTilesSource.createDataSource(metadataUrl, {});metadata = await tileSource.metadata;}@@ -85,7 +72,22 @@ export default function useFetchVectorTileMetadata({if (!metadata) {throw new Error('Failed to fetch metadata');}- setProcessedData(metadata);++ const processedMetadata = process(metadata);+ if (processedMetadata instanceof Error) {+ setError(processedMetadata);+ } else {+ setError(null);++ await getFieldsFromTile({+ remoteTileFormat,+ tilesetUrl,+ metadataUrl,+ metadata: processedMetadata+ });++ setData(processedMetadata);+ }} catch (metadataError) {setError(metadataError as any);}@@ -94,7 +96,7 @@ export default function useFetchVectorTileMetadata({};getAndProcessMetadata();- }, [url, remoteTileFormat, setProcessedData]);+ }, [metadataUrl, tilesetUrl, remoteTileFormat, setError, setData, process]);return {data, loading, error};}diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsxindex 70e9ceebf..a02e3bec9 100644--- a/src/components/src/modal-container.tsx+++ b/src/components/src/modal-container.tsx@@ -194,7 +194,8 @@ export default function ModalContainerFactory(disableDataOperation: true},{- autoCreateLayers: true+ autoCreateLayers: true,+ centerMap: true});this._closeModal();diff --git a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsxindex 3ec5db7b2..53690e813 100644--- a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx+++ b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx@@ -95,10 +95,13 @@ const TilesetVectorForm: React.FC<TilesetVectorFormProps> = ({setResponse}) => {event.preventDefault();const newTileUrl = event.target.value;setTileUrl(newTileUrl);- const potentialMetadataUrl = isPMTilesUrl(newTileUrl) ? newTileUrl : getMetaUrl(newTileUrl);++ const usePMTiles = isPMTilesUrl(newTileUrl);+ const potentialMetadataUrl = usePMTiles ? newTileUrl : getMetaUrl(newTileUrl);if (!metadataUrl && potentialMetadataUrl) {// check if URL exists before setting it as the metadata URL- const resp = await fetch(potentialMetadataUrl);+ // Note: The {method: HEAD} request often fails, likely due to individual storage settings.+ const resp = usePMTiles ? ({ok: true} as Response) : await fetch(potentialMetadataUrl);if (resp.ok) {setInitialFetchError(null);setMetadataUrl(potentialMetadataUrl);@@ -127,11 +130,17 @@ const TilesetVectorForm: React.FC<TilesetVectorFormProps> = ({setResponse}) => {loading,error: metaError} = useFetchVectorTileMetadata({- url: metadataUrl,+ metadataUrl,+ tilesetUrl: tileUrl,remoteTileFormat: isPMTilesUrl(metadataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,process});+ // reset initial fetch error if the metadata is available+ if (metadata && initialFetchError) {+ setInitialFetchError(null);+ }+useEffect(() => {if (tileName && tileUrl) {const dataset = getDatasetAttributesFromVectorTile({diff --git a/src/layers/src/vector-tile/abstract-tile-layer.ts b/src/layers/src/vector-tile/abstract-tile-layer.tsindex 0c6bbfe22..a975ae1fe 100644--- a/src/layers/src/vector-tile/abstract-tile-layer.ts+++ b/src/layers/src/vector-tile/abstract-tile-layer.ts@@ -65,6 +65,7 @@ export type AbstractTileLayerVisConfigSettings = {export type LayerData = {minZoom?: number;maxZoom?: number;+ bounds?: number[];getPointRadius?: () => number;};@@ -408,10 +409,16 @@ export default abstract class AbstractTileLayer<});const metadata = dataset?.metadata as LayerData | undefined;+ const metadataToData = metadata+ ? {+ minZoom: metadata.minZoom,+ maxZoom: metadata.maxZoom,+ bounds: metadata.bounds+ }+ : {};return {- ...(metadata ? {minZoom: metadata.minZoom} : {}),- ...(metadata ? {maxZoom: metadata.maxZoom} : {}),+ ...metadataToData,...accessors};}diff --git a/src/layers/src/vector-tile/vector-tile-layer.ts b/src/layers/src/vector-tile/vector-tile-layer.tsindex 15253f123..879a77373 100644--- a/src/layers/src/vector-tile/vector-tile-layer.ts+++ b/src/layers/src/vector-tile/vector-tile-layer.ts@@ -5,7 +5,7 @@ import {FeatureCollection, Feature} from 'geojson';import {Layer as DeckLayer} from '@deck.gl/core/typed';import {_Tile2DHeader as Tile2DHeader} from '@deck.gl/geo-layers/typed';-import {GeoJsonLayer} from '@deck.gl/layers/typed';+import {GeoJsonLayer, PathLayer} from '@deck.gl/layers/typed';import {MVTSource, MVTTileSource} from '@loaders.gl/mvt';import {PMTilesSource, PMTilesTileSource} from '@loaders.gl/pmtiles';import GL from '@luma.gl/constants';@@ -175,6 +175,35 @@ export type VectorTileLayerVisConfigSettings = Merge<}>;+export function tileLayerBoundsLayer(id: string, props: {bounds?: number[]}): DeckLayer[] {+ const {bounds} = props;+ if (bounds?.length !== 4) return [];++ const data = [+ {+ path: [+ [bounds[0], bounds[1]],+ [bounds[2], bounds[1]],+ [bounds[2], bounds[3]],+ [bounds[0], bounds[3]],+ [bounds[0], bounds[1]]+ ]+ }+ ];++ const layer = new PathLayer({+ id: `${id}-vector-tile-bounds`,+ data,+ getPath: d => d.path,+ getColor: [128, 128, 128, 255],+ getWidth: 1,+ widthUnits: 'pixels',+ pickable: false+ });++ return [layer];+}+export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Feature[]> {declare config: VectorTileLayerConfig;declare visConfigSettings: VectorTileLayerVisConfigSettings;@@ -625,6 +654,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu})]: [])+ // ...tileLayerBoundsLayer(defaultLayerProps.id, data),];return layers;diff --git a/src/table/src/dataset-utils.ts b/src/table/src/dataset-utils.tsindex a8060793d..47ba97356 100644--- a/src/table/src/dataset-utils.ts+++ b/src/table/src/dataset-utils.ts@@ -17,7 +17,7 @@ import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';import {getMVTMetadata} from './tileset/tileset-utils';-import {parseVectorMetadata} from './tileset/vector-tile-utils';+import {parseVectorMetadata, getFieldsFromTile} from './tileset/vector-tile-utils';// apply a color for each dataset// to use as label colors@@ -137,12 +137,13 @@ async function refreshRemoteData(datasetInfo: CreateTableProps) {return null;}- const {remoteTileFormat, tilesetMetadataUrl} =+ const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =(datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};if (!(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||- typeof tilesetMetadataUrl !== 'string'+ typeof tilesetMetadataUrl !== 'string' ||+ typeof tilesetDataUrl !== 'string') {return null;}@@ -157,7 +158,16 @@ async function refreshRemoteData(datasetInfo: CreateTableProps) {}if (rawMetadata) {- return parseVectorMetadata(rawMetadata);+ const metadata = parseVectorMetadata(rawMetadata);++ await getFieldsFromTile({+ remoteTileFormat,+ tilesetUrl: tilesetDataUrl,+ metadataUrl: tilesetMetadataUrl,+ metadata+ });++ return metadata;}} catch (err) {// ignore for now, and use old metadata?diff --git a/src/table/src/tileset/vector-tile-utils.ts b/src/table/src/tileset/vector-tile-utils.tsindex 6c92bbeb7..3716d0947 100644--- a/src/table/src/tileset/vector-tile-utils.ts+++ b/src/table/src/tileset/vector-tile-utils.ts@@ -1,30 +1,34 @@// SPDX-License-Identifier: MIT// Copyright contributors to the kepler.gl project+import {DATA_TYPES as ANALYZER_DATA_TYPES} from 'type-analyzer';import {ascending} from 'd3-array';import Console from 'global/console';import uniq from 'lodash.uniq';import {DATA_TYPES} from 'type-analyzer';-import {TileJSON} from '@loaders.gl/mvt';-import {PMTilesMetadata} from '@loaders.gl/pmtiles';+import {MVTSource, TileJSON} from '@loaders.gl/mvt';+import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';import {analyzerTypeToFieldType,containValidTime,notNullorUndefined as notNullOrUndefined,- parseUri+ parseUri,+ getFieldsFromData} from '@kepler.gl/common-utils';-import {DatasetType, ALL_FIELD_TYPES, FILTER_TYPES} from '@kepler.gl/constants';+import {DatasetType, ALL_FIELD_TYPES, FILTER_TYPES, RemoteTileFormat} from '@kepler.gl/constants';++import {Feature, Field as KeplerField, KeplerLayer} from '@kepler.gl/types';+import {clamp, formatNumberByStep, getNumericStepSize, timeToUnixMilli} from '@kepler.gl/utils';+import {FilterProps,NumericFieldFilterProps,BooleanFieldFilterProps,StringFieldFilterProps,- KeplerTable as KeplerDataset-} from '@kepler.gl/table';-import {Field as KeplerField, KeplerLayer} from '@kepler.gl/types';-import {clamp, formatNumberByStep, getNumericStepSize, timeToUnixMilli} from '@kepler.gl/utils';+ default as KeplerDataset+} from '../kepler-table';export const getLoaderOptions = () => {return {@@ -667,3 +671,122 @@ export function matchDatasetType(dataset: KeplerDataset, layer: KeplerLayer): bolayer.supportedDatasetTypes.includes(dataset.type || ''));}++type GetFieldsFromTileProps = {+ remoteTileFormat: RemoteTileFormat;+ tilesetUrl: string | null;+ metadataUrl: string | null;+ metadata: VectorTileMetadata | null;+};++/**+ * Extracts fields from a tile and updates the metadata object with found fields.+ * Note: this function is for tilesets that don't include fields in metadata (most likely saved with older spec.).+ * @param params.remoteTileFormat The format of the remote tile (MVT or PMTiles).+ * @param params.tilesetUrl The URL of the tileset.+ * @param params.metadataUrl The URL of the metadata.+ * @param params.metadata The metadata object containing fields and tile properties.+ */+export const getFieldsFromTile = async ({+ remoteTileFormat,+ tilesetUrl,+ metadataUrl,+ metadata+}: GetFieldsFromTileProps) => {+ try {+ if (+ tilesetUrl &&+ metadataUrl &&+ metadata &&+ metadata.fields?.length === 0 &&+ metadata.minZoom &&+ metadata.bounds?.length === 4+ ) {+ const lon = (metadata.bounds[0] + metadata.bounds[2]) / 2;+ const lat = (metadata.bounds[1] + metadata.bounds[3]) / 2;+ const tileIndices = lonLatToTileIndex(lon, lat, metadata.minZoom);++ const tileSource =+ remoteTileFormat === RemoteTileFormat.MVT+ ? MVTSource.createDataSource(decodeURIComponent(tilesetUrl), {+ mvt: {+ metadataUrl: decodeURIComponent(metadataUrl)+ }+ })+ : PMTilesSource.createDataSource(tilesetUrl, {});+ const tile = await tileSource.getTileData({index: tileIndices} as any);+ const updatedFields = tileToFields(tile).map(f => {+ return {+ ...f,+ analyzerType: f.analyzerType || ALL_FIELD_TYPES.string,+ id: f.id || f.name+ };+ });++ metadata.fields = updatedFields;+ }+ } catch {+ // ignore, as this is experimental fallback+ }+};++/**+ * Converts tile features into Kepler fields.+ * @param tile The tile object containing features.+ * @returns An array of Kepler fields derived from the tile's features.+ */+const tileToFields = (tile: {features: Feature[]}): KeplerField[] => {+ if (tile?.features?.length > 0) {+ const header = Object.keys(tile.features[0].properties);+ const output = tile.features.map(f => {+ const obj = {};+ header.forEach(columnName => {+ obj[columnName] = f.properties[columnName];+ });+ return obj;+ });++ const fields = getFieldsFromData(output, header);+ // extra transformation of strings to numbers for tiles isn't implemented, so use string, not computed types+ return fields.map(f => {+ const forceString =+ (f.type === 'integer' || f.type === 'float') &&+ typeof tile.features[0].properties[f.name] === 'string';++ return {+ ...f,+ analyzerType: forceString ? ANALYZER_DATA_TYPES.STRING : f.analyzerType,+ type: forceString ? 'string' : f.type+ };+ });+ }+ return [];+};++/**+ * Converts longitude, latitude, and zoom level into vector tile indices (x, y, z).+ * @param lon Longitude in degrees, ranging from -180 to 180.+ * @param lat Latitude in degrees, ranging from -90 to 90.+ * @param zoom Zoom level (integer), where higher values provide more detail.+ * @returns Tile indices with x and y coordinates and zoom level z.+ */+function lonLatToTileIndex(lon: number, lat: number, zoom: number) {+ if (lat < -85.0511 || lat > 85.0511) {+ throw new Error('Latitude out of range. Must be between -85.0511 and 85.0511.');+ }+ if (zoom < 0) {+ throw new Error('Zoom level must be a non-negative integer.');+ }++ // 2^zoom (number of tiles per axis at given zoom level)+ const scale = 1 << zoom;+ // Convert longitude to tile X+ const x = Math.floor(((lon + 180) / 360) * scale);+ // Convert latitude to tile Y+ const latRad = (lat * Math.PI) / 180;+ const y = Math.floor(+ ((1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2) * scale+ );++ return {x, y, z: zoom};+}