Skip to content

Kepler.gl is a powerful open source geospatial analysis tool for large-scale data sets.

[fix] fixes for vector-tile layer (#3013)

Igor Dykhta7107e41776fb
diff --git a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts b/src/components/src/hooks/use-fetch-vector-tile-metadata.ts
index b996c506c..6c59042d4 100644
--- a/src/components/src/hooks/use-fetch-vector-tile-metadata.ts
+++ b/src/components/src/hooks/use-fetch-vector-tile-metadata.ts
@@ -1,16 +1,17 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
-import {useCallback, useEffect, useState} from 'react';
+import {useEffect, useState} from 'react';
-import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
+import {TileJSON} from '@loaders.gl/mvt';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {RemoteTileFormat} from '@kepler.gl/constants';
-import {getMVTMetadata, VectorTileMetadata} from '@kepler.gl/table';
+import {getMVTMetadata, VectorTileMetadata, getFieldsFromTile} from '@kepler.gl/table';
type FetchVectorTileMetadataProps = {
- url: string | null;
+ metadataUrl: string | null;
+ tilesetUrl: string | null;
remoteTileFormat: RemoteTileFormat;
process?: (json: PMTilesMetadata | TileJSON) => VectorTileMetadata | Error | null;
};
@@ -36,39 +37,25 @@ type FetchVectorTileMetadataReturn = {
/** Hook to fetch and return mvt or pmtiles metadata. */
export default function useFetchVectorTileMetadata({
remoteTileFormat,
- url,
+ tilesetUrl,
+ metadataUrl,
process = DEFAULT_PROCESS_FUNCTION
}: FetchVectorTileMetadataProps): FetchVectorTileMetadataReturn {
const [error, setError] = useState<Error | null>(null);
const [data, setData] = useState<VectorTileMetadata | null>(null);
const [loading, setLoading] = useState<boolean>(false);
- const setProcessedData = useCallback(
- (value: PMTilesMetadata | TileJSON | null) => {
- if (!value) {
- return setError(value);
- }
- const processedData = process(value);
- if (processedData instanceof Error) {
- setError(processedData);
- } else {
- setData(processedData);
- }
- },
- [setError, setData, process]
- );
-
useEffect(() => {
const getAndProcessMetadata = async () => {
setError(null);
setData(null);
- if (url) {
+ if (metadataUrl) {
setLoading(true);
try {
let metadata: PMTilesMetadata | TileJSON | null = null;
if (remoteTileFormat === RemoteTileFormat.MVT) {
- metadata = await getMVTMetadata(url);
+ metadata = await getMVTMetadata(metadataUrl);
// MVTSource returns messy partial metadata
// MVTSource.createDataSource('', {
@@ -77,7 +64,7 @@ export default function useFetchVectorTileMetadata({
// }
// })
} else {
- const tileSource = PMTilesSource.createDataSource(url, {});
+ const tileSource = PMTilesSource.createDataSource(metadataUrl, {});
metadata = await tileSource.metadata;
}
@@ -85,7 +72,22 @@ export default function useFetchVectorTileMetadata({
if (!metadata) {
throw new Error('Failed to fetch metadata');
}
- setProcessedData(metadata);
+
+ const processedMetadata = process(metadata);
+ if (processedMetadata instanceof Error) {
+ setError(processedMetadata);
+ } else {
+ setError(null);
+
+ await getFieldsFromTile({
+ remoteTileFormat,
+ tilesetUrl,
+ metadataUrl,
+ metadata: processedMetadata
+ });
+
+ setData(processedMetadata);
+ }
} catch (metadataError) {
setError(metadataError as any);
}
@@ -94,7 +96,7 @@ export default function useFetchVectorTileMetadata({
};
getAndProcessMetadata();
- }, [url, remoteTileFormat, setProcessedData]);
+ }, [metadataUrl, tilesetUrl, remoteTileFormat, setError, setData, process]);
return {data, loading, error};
}
diff --git a/src/components/src/modal-container.tsx b/src/components/src/modal-container.tsx
index 70e9ceebf..a02e3bec9 100644
--- a/src/components/src/modal-container.tsx
+++ b/src/components/src/modal-container.tsx
@@ -194,7 +194,8 @@ export default function ModalContainerFactory(
disableDataOperation: true
},
{
- autoCreateLayers: true
+ autoCreateLayers: true,
+ centerMap: true
}
);
this._closeModal();
diff --git a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx
index 3ec5db7b2..53690e813 100644
--- a/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx
+++ b/src/components/src/modals/tilesets-modals/tileset-vector-form.tsx
@@ -95,10 +95,13 @@ const TilesetVectorForm: React.FC<TilesetVectorFormProps> = ({setResponse}) => {
event.preventDefault();
const newTileUrl = event.target.value;
setTileUrl(newTileUrl);
- const potentialMetadataUrl = isPMTilesUrl(newTileUrl) ? newTileUrl : getMetaUrl(newTileUrl);
+
+ const usePMTiles = isPMTilesUrl(newTileUrl);
+ const potentialMetadataUrl = usePMTiles ? newTileUrl : getMetaUrl(newTileUrl);
if (!metadataUrl && potentialMetadataUrl) {
// check if URL exists before setting it as the metadata URL
- const resp = await fetch(potentialMetadataUrl);
+ // Note: The {method: HEAD} request often fails, likely due to individual storage settings.
+ const resp = usePMTiles ? ({ok: true} as Response) : await fetch(potentialMetadataUrl);
if (resp.ok) {
setInitialFetchError(null);
setMetadataUrl(potentialMetadataUrl);
@@ -127,11 +130,17 @@ const TilesetVectorForm: React.FC<TilesetVectorFormProps> = ({setResponse}) => {
loading,
error: metaError
} = useFetchVectorTileMetadata({
- url: metadataUrl,
+ metadataUrl,
+ tilesetUrl: tileUrl,
remoteTileFormat: isPMTilesUrl(metadataUrl) ? RemoteTileFormat.PMTILES : RemoteTileFormat.MVT,
process
});
+ // reset initial fetch error if the metadata is available
+ if (metadata && initialFetchError) {
+ setInitialFetchError(null);
+ }
+
useEffect(() => {
if (tileName && tileUrl) {
const dataset = getDatasetAttributesFromVectorTile({
diff --git a/src/layers/src/vector-tile/abstract-tile-layer.ts b/src/layers/src/vector-tile/abstract-tile-layer.ts
index 0c6bbfe22..a975ae1fe 100644
--- a/src/layers/src/vector-tile/abstract-tile-layer.ts
+++ b/src/layers/src/vector-tile/abstract-tile-layer.ts
@@ -65,6 +65,7 @@ export type AbstractTileLayerVisConfigSettings = {
export type LayerData = {
minZoom?: number;
maxZoom?: number;
+ bounds?: number[];
getPointRadius?: () => number;
};
@@ -408,10 +409,16 @@ export default abstract class AbstractTileLayer<
});
const metadata = dataset?.metadata as LayerData | undefined;
+ const metadataToData = metadata
+ ? {
+ minZoom: metadata.minZoom,
+ maxZoom: metadata.maxZoom,
+ bounds: metadata.bounds
+ }
+ : {};
return {
- ...(metadata ? {minZoom: metadata.minZoom} : {}),
- ...(metadata ? {maxZoom: metadata.maxZoom} : {}),
+ ...metadataToData,
...accessors
};
}
diff --git a/src/layers/src/vector-tile/vector-tile-layer.ts b/src/layers/src/vector-tile/vector-tile-layer.ts
index 15253f123..879a77373 100644
--- a/src/layers/src/vector-tile/vector-tile-layer.ts
+++ b/src/layers/src/vector-tile/vector-tile-layer.ts
@@ -5,7 +5,7 @@ import {FeatureCollection, Feature} from 'geojson';
import {Layer as DeckLayer} from '@deck.gl/core/typed';
import {_Tile2DHeader as Tile2DHeader} from '@deck.gl/geo-layers/typed';
-import {GeoJsonLayer} from '@deck.gl/layers/typed';
+import {GeoJsonLayer, PathLayer} from '@deck.gl/layers/typed';
import {MVTSource, MVTTileSource} from '@loaders.gl/mvt';
import {PMTilesSource, PMTilesTileSource} from '@loaders.gl/pmtiles';
import GL from '@luma.gl/constants';
@@ -175,6 +175,35 @@ export type VectorTileLayerVisConfigSettings = Merge<
}
>;
+export function tileLayerBoundsLayer(id: string, props: {bounds?: number[]}): DeckLayer[] {
+ const {bounds} = props;
+ if (bounds?.length !== 4) return [];
+
+ const data = [
+ {
+ path: [
+ [bounds[0], bounds[1]],
+ [bounds[2], bounds[1]],
+ [bounds[2], bounds[3]],
+ [bounds[0], bounds[3]],
+ [bounds[0], bounds[1]]
+ ]
+ }
+ ];
+
+ const layer = new PathLayer({
+ id: `${id}-vector-tile-bounds`,
+ data,
+ getPath: d => d.path,
+ getColor: [128, 128, 128, 255],
+ getWidth: 1,
+ widthUnits: 'pixels',
+ pickable: false
+ });
+
+ return [layer];
+}
+
export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Feature[]> {
declare config: VectorTileLayerConfig;
declare visConfigSettings: VectorTileLayerVisConfigSettings;
@@ -625,6 +654,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
})
]
: [])
+ // ...tileLayerBoundsLayer(defaultLayerProps.id, data),
];
return layers;
diff --git a/src/table/src/dataset-utils.ts b/src/table/src/dataset-utils.ts
index a8060793d..47ba97356 100644
--- a/src/table/src/dataset-utils.ts
+++ b/src/table/src/dataset-utils.ts
@@ -17,7 +17,7 @@ import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
import {getMVTMetadata} from './tileset/tileset-utils';
-import {parseVectorMetadata} from './tileset/vector-tile-utils';
+import {parseVectorMetadata, getFieldsFromTile} from './tileset/vector-tile-utils';
// apply a color for each dataset
// to use as label colors
@@ -137,12 +137,13 @@ async function refreshRemoteData(datasetInfo: CreateTableProps) {
return null;
}
- const {remoteTileFormat, tilesetMetadataUrl} =
+ const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
(datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};
if (
!(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
- typeof tilesetMetadataUrl !== 'string'
+ typeof tilesetMetadataUrl !== 'string' ||
+ typeof tilesetDataUrl !== 'string'
) {
return null;
}
@@ -157,7 +158,16 @@ async function refreshRemoteData(datasetInfo: CreateTableProps) {
}
if (rawMetadata) {
- return parseVectorMetadata(rawMetadata);
+ const metadata = parseVectorMetadata(rawMetadata);
+
+ await getFieldsFromTile({
+ remoteTileFormat,
+ tilesetUrl: tilesetDataUrl,
+ metadataUrl: tilesetMetadataUrl,
+ metadata
+ });
+
+ return metadata;
}
} catch (err) {
// ignore for now, and use old metadata?
diff --git a/src/table/src/tileset/vector-tile-utils.ts b/src/table/src/tileset/vector-tile-utils.ts
index 6c92bbeb7..3716d0947 100644
--- a/src/table/src/tileset/vector-tile-utils.ts
+++ b/src/table/src/tileset/vector-tile-utils.ts
@@ -1,30 +1,34 @@
// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project
+import {DATA_TYPES as ANALYZER_DATA_TYPES} from 'type-analyzer';
import {ascending} from 'd3-array';
import Console from 'global/console';
import uniq from 'lodash.uniq';
import {DATA_TYPES} from 'type-analyzer';
-import {TileJSON} from '@loaders.gl/mvt';
-import {PMTilesMetadata} from '@loaders.gl/pmtiles';
+import {MVTSource, TileJSON} from '@loaders.gl/mvt';
+import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {
analyzerTypeToFieldType,
containValidTime,
notNullorUndefined as notNullOrUndefined,
- parseUri
+ parseUri,
+ getFieldsFromData
} from '@kepler.gl/common-utils';
-import {DatasetType, ALL_FIELD_TYPES, FILTER_TYPES} from '@kepler.gl/constants';
+import {DatasetType, ALL_FIELD_TYPES, FILTER_TYPES, RemoteTileFormat} from '@kepler.gl/constants';
+
+import {Feature, Field as KeplerField, KeplerLayer} from '@kepler.gl/types';
+import {clamp, formatNumberByStep, getNumericStepSize, timeToUnixMilli} from '@kepler.gl/utils';
+
import {
FilterProps,
NumericFieldFilterProps,
BooleanFieldFilterProps,
StringFieldFilterProps,
- KeplerTable as KeplerDataset
-} from '@kepler.gl/table';
-import {Field as KeplerField, KeplerLayer} from '@kepler.gl/types';
-import {clamp, formatNumberByStep, getNumericStepSize, timeToUnixMilli} from '@kepler.gl/utils';
+ default as KeplerDataset
+} from '../kepler-table';
export const getLoaderOptions = () => {
return {
@@ -667,3 +671,122 @@ export function matchDatasetType(dataset: KeplerDataset, layer: KeplerLayer): bo
layer.supportedDatasetTypes.includes(dataset.type || '')
);
}
+
+type GetFieldsFromTileProps = {
+ remoteTileFormat: RemoteTileFormat;
+ tilesetUrl: string | null;
+ metadataUrl: string | null;
+ metadata: VectorTileMetadata | null;
+};
+
+/**
+ * Extracts fields from a tile and updates the metadata object with found fields.
+ * Note: this function is for tilesets that don't include fields in metadata (most likely saved with older spec.).
+ * @param params.remoteTileFormat The format of the remote tile (MVT or PMTiles).
+ * @param params.tilesetUrl The URL of the tileset.
+ * @param params.metadataUrl The URL of the metadata.
+ * @param params.metadata The metadata object containing fields and tile properties.
+ */
+export const getFieldsFromTile = async ({
+ remoteTileFormat,
+ tilesetUrl,
+ metadataUrl,
+ metadata
+}: GetFieldsFromTileProps) => {
+ try {
+ if (
+ tilesetUrl &&
+ metadataUrl &&
+ metadata &&
+ metadata.fields?.length === 0 &&
+ metadata.minZoom &&
+ metadata.bounds?.length === 4
+ ) {
+ const lon = (metadata.bounds[0] + metadata.bounds[2]) / 2;
+ const lat = (metadata.bounds[1] + metadata.bounds[3]) / 2;
+ const tileIndices = lonLatToTileIndex(lon, lat, metadata.minZoom);
+
+ const tileSource =
+ remoteTileFormat === RemoteTileFormat.MVT
+ ? MVTSource.createDataSource(decodeURIComponent(tilesetUrl), {
+ mvt: {
+ metadataUrl: decodeURIComponent(metadataUrl)
+ }
+ })
+ : PMTilesSource.createDataSource(tilesetUrl, {});
+ const tile = await tileSource.getTileData({index: tileIndices} as any);
+ const updatedFields = tileToFields(tile).map(f => {
+ return {
+ ...f,
+ analyzerType: f.analyzerType || ALL_FIELD_TYPES.string,
+ id: f.id || f.name
+ };
+ });
+
+ metadata.fields = updatedFields;
+ }
+ } catch {
+ // ignore, as this is experimental fallback
+ }
+};
+
+/**
+ * Converts tile features into Kepler fields.
+ * @param tile The tile object containing features.
+ * @returns An array of Kepler fields derived from the tile's features.
+ */
+const tileToFields = (tile: {features: Feature[]}): KeplerField[] => {
+ if (tile?.features?.length > 0) {
+ const header = Object.keys(tile.features[0].properties);
+ const output = tile.features.map(f => {
+ const obj = {};
+ header.forEach(columnName => {
+ obj[columnName] = f.properties[columnName];
+ });
+ return obj;
+ });
+
+ const fields = getFieldsFromData(output, header);
+ // extra transformation of strings to numbers for tiles isn't implemented, so use string, not computed types
+ return fields.map(f => {
+ const forceString =
+ (f.type === 'integer' || f.type === 'float') &&
+ typeof tile.features[0].properties[f.name] === 'string';
+
+ return {
+ ...f,
+ analyzerType: forceString ? ANALYZER_DATA_TYPES.STRING : f.analyzerType,
+ type: forceString ? 'string' : f.type
+ };
+ });
+ }
+ return [];
+};
+
+/**
+ * Converts longitude, latitude, and zoom level into vector tile indices (x, y, z).
+ * @param lon Longitude in degrees, ranging from -180 to 180.
+ * @param lat Latitude in degrees, ranging from -90 to 90.
+ * @param zoom Zoom level (integer), where higher values provide more detail.
+ * @returns Tile indices with x and y coordinates and zoom level z.
+ */
+function lonLatToTileIndex(lon: number, lat: number, zoom: number) {
+ if (lat < -85.0511 || lat > 85.0511) {
+ throw new Error('Latitude out of range. Must be between -85.0511 and 85.0511.');
+ }
+ if (zoom < 0) {
+ throw new Error('Zoom level must be a non-negative integer.');
+ }
+
+ // 2^zoom (number of tiles per axis at given zoom level)
+ const scale = 1 << zoom;
+ // Convert longitude to tile X
+ const x = Math.floor(((lon + 180) / 360) * scale);
+ // Convert latitude to tile Y
+ const latRad = (lat * Math.PI) / 180;
+ const y = Math.floor(
+ ((1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2) * scale
+ );
+
+ return {x, y, z: zoom};
+}