Interactive interface for browsing global, full-resolution satellite imagery
WV-3400 additional changes to improve build speed + add progress bar
ZachTRicec662641a12f1
diff --git a/tasks/build-options/extractConfigFromWMTS.js b/tasks/build-options/extractConfigFromWMTS.jsindex a89efe2f0..ee8f6e262 100644--- a/tasks/build-options/extractConfigFromWMTS.js+++ b/tasks/build-options/extractConfigFromWMTS.js@@ -64,42 +64,147 @@ if (!fs.existsSync(outputDir)) {const tolerant = config.tolerantconst entries = config['wv-options-wmts']-const skip = config.skip || []+const skipSet = new Set(config.skip || [])+const skipPalettesSet = new Set(Object.keys(config.skipPalettes || {}))let totalLayerCount = 0let totalWarningCount = 0let totalErrorCount = 0-class SkipException extends Error {- constructor (message) {- super(message)- this.name = 'SkipException'+// Concurrency + caching utilities+const layerConcurrency = parseInt(process.env.LAYER_CONCURRENCY || '8', 10)+// Keep a stable reference to the real fetch to avoid recursion+const nativeFetch = globalThis.fetch ? globalThis.fetch.bind(globalThis) : undefined+// Cache fetched text, not Response, to avoid body reuse problems+const describeDomainsTextCache = new Map()++// Progress tracking+const isTTY = process.stderr.isTTY+const progress = {+ total: 0,+ done: 0,+ lastRender: 0+}+function renderProgress (force = false) {+ if (!isTTY) return+ const now = Date.now()+ // Throttle to ~20fps or on force+ if (!force && now - progress.lastRender < 50) return+ progress.lastRender = now+ const line = `${prog}: ${progress.done}/${progress.total} layers processed`+ // Clear line and rewrite (works in most terminals)+ process.stderr.write(`\r${line}`)+}+function finishProgressLine () {+ if (!isTTY) return+ process.stderr.write('\n')+}++async function cachedFetch (url, opts) {+ const eligible = typeof url === 'string' && url.includes('/wmts/') && url.endsWith('.xml')+ if (!eligible || !nativeFetch) {+ return nativeFetch ? nativeFetch(url, opts) : Promise.reject(new Error('fetch not available'))+ }++ if (describeDomainsTextCache.has(url)) {+ const textPromise = describeDomainsTextCache.get(url)+ return {+ ok: true,+ text: () => textPromise+ }+ }++ // Perform the real fetch once, cache the resulting text+ const responseTextPromise = (async () => {+ try {+ const res = await nativeFetch(url, opts)+ if (!res || !res.ok) return ''+ return res.text()+ } catch {+ return ''+ }+ })()++ describeDomainsTextCache.set(url, responseTextPromise)++ const res = await nativeFetch(url, opts)+ const ok = !!(res && res.ok)+ if (ok) {+ // Ensure any rejection on the cached promise is observed to avoid unhandled rejections+ responseTextPromise.catch(() => {})+ }+ return {+ ok,+ text: () => responseTextPromise}}-/**- * Main function- * @returns {Promise<void>}- * @throws {Error}- * @throws {SkipException}- */+async function asyncPool (limit, array, iteratorFn) {+ const ret = []+ const executing = []+ for (const item of array) {+ const p = Promise.resolve().then(() => iteratorFn(item))+ ret.push(p)+ if (limit <= array.length) {+ const e = p.then(() => {+ executing.splice(executing.indexOf(e), 1)+ }).catch(() => {+ executing.splice(executing.indexOf(e), 1)+ })+ executing.push(e)+ if (executing.length >= limit) {+ await Promise.race(executing)+ }+ }+ }+ return Promise.allSettled(ret)+}++// Pre-scan all entries to compute a stable total layer count for progress+async function computeTotalLayers (entriesList) {+ let total = 0+ const tasks = entriesList.map(async (entry) => {+ try {+ const inputFile = path.join(inputDir, entry.from)+ const xml = await fs.promises.readFile(inputFile, 'utf8')+ const gc = JSON.parse(convert.xml2json(xml, { compact: true }))+ const count = Array.isArray(gc?.Capabilities?.Contents?.Layer)+ ? gc.Capabilities.Contents.Layer.length+ : (gc?.Capabilities?.Contents?.Layer ? 1 : 0)+ total += count+ } catch {+ // Ignore count for entries that fail to parse in prescan+ }+ })+ await Promise.allSettled(tasks)+ return total+}+async function main () {console.warn(`${prog}: Processing ${entries.length} entries...`)- const promises = entries.map((entry) => processEntry(entry))- const results = await Promise.allSettled(promises)+ // Setup a stable total for progress display+ progress.total = await computeTotalLayers(entries)+ if (progress.total > 0) renderProgress(true)++ const entryResults = await Promise.allSettled(entries.map(e => processEntry(e)))- for (const { status, value } of results) {- if (status === 'rejected' || !value) continue+ finishProgressLine()++ for (const { status, value } of entryResults) {+ if (status !== 'fulfilled' || !value) continueconst { errorCount, warningCount, layerCount, wv, to, source } = value- console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${source}`)+ if (mode === 'verbose') {+ console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${source}`)+ }const outputFile = path.join(outputDir, to)- await fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {- if (err) {- console.error(err)- }- })+ try {+ await fs.promises.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8')+ } catch (err) {+ console.error(`${prog}: ERROR writing ${outputFile}: ${err}`)+ totalErrorCount += 1+ }totalErrorCount += errorCounttotalWarningCount += warningCount@@ -114,10 +219,10 @@ async function main () {}async function processEntry (entry) {- console.warn(`${prog}: Processing ${entry.source} from ${entry.from} --> ${entry.to}...`)- const wv = {}- wv.layers = {}- wv.sources = {}+ if (mode === 'verbose') {+ console.warn(`${prog}: Processing ${entry.source} from ${entry.from} --> ${entry.to}...`)+ }+ const wv = { layers: {}, sources: {} }let wvMatrixSets = {}let layerCount = 0@@ -129,49 +234,76 @@ async function processEntry (entry) {let gctry {const xml = await fs.promises.readFile(inputFile, 'utf8')- gc = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))+ // Remove pretty spaces to reduce parse overhead+ gc = JSON.parse(convert.xml2json(xml, { compact: true }))} catch (e) {+ const msg = `${prog}: [${inputFile}] Unable to get GC: ${e}`if (tolerant) {- console.warn(`${prog}: WARN: [${inputFile}] Unable to get GC: ${e}\n`)+ console.warn(`${prog}: WARN: ${msg}`)warningCount += 1} else {- console.error(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)+ console.error(`${prog}: ERROR: ${msg}`)errorCount += 1}return { errorCount, warningCount, layerCount }}- const gcContents = gc.Capabilities.Contents+ const gcContents = gc?.Capabilities?.Contentsconst wvLayers = wv.layersif (!gcContents || !gcContents.Layer) {errorCount += 1- console.error(`${prog}: ERROR: [${gcId}] No layers\n`)+ console.error(`${prog}: ERROR: [${gcId}] No layers`)return { errorCount, warningCount, layerCount }}- console.warn(`${prog}: Processing ${gcContents.Layer.length} layers for ${entry.source}...`)- const promises = gcContents.Layer.map((gcLayer) => processLayer(gcLayer, wvLayers, entry))- const results = await Promise.allSettled(promises)- console.warn(`${prog}: Processed ${results.length} layers for ${entry.source}...`)+ const layersArr = Array.isArray(gcContents.Layer) ? gcContents.Layer : [gcContents.Layer]+ if (mode === 'verbose') {+ console.warn(`${prog}: Layer count for ${entry.source}: ${layersArr.length}`)+ }- layerCount += results.length+ // Temporarily override fetch with caching layer for temporal requests+ const originalFetch = globalThis.fetch+ globalThis.fetch = cachedFetch++ const layerResults = await asyncPool(layerConcurrency, layersArr, async (gcLayer) => {+ try {+ const res = await processLayer(gcLayer, wvLayers, entry)+ return res+ } finally {+ // Increment and render progress for every finished layer+ progress.done += 1+ if (progress.done <= progress.total) renderProgress()+ }+ })- for (const result of results) {- if (result.status === 'fulfilled') continue- if (result.reason instanceof SkipException) {- warningCount += 1- console.warn(`${prog}: WARNING: Skipping\n`)- } else {- errorCount += 1- console.error(result.reason.stack)- console.error(`${prog}: ERROR: ${result.reason}\n`)+ globalThis.fetch = originalFetch++ layerCount += layerResults.length++ for (const res of layerResults) {+ if (res.status === 'fulfilled') {+ const val = res.value+ if (val && val.skipped) {+ warningCount += 1+ if (mode === 'verbose') {+ console.warn(`${prog}: WARNING: Skipping layer`)+ }+ }+ continue+ }+ // Real error+ errorCount += 1+ if (mode === 'verbose') {+ console.error(res.reason?.stack || res.reason)+ console.error(`${prog}: ERROR: ${res.reason}`)}}const gcTileMatrixSet = gcContents.TileMatrixSet-- wvMatrixSets = Array.isArray(gcTileMatrixSet) ? gcTileMatrixSet.map(gcMatrixSet => processMatrixSet(gcMatrixSet, entry)) : [processMatrixSet(gcTileMatrixSet, entry)]+ wvMatrixSets = Array.isArray(gcTileMatrixSet)+ ? gcTileMatrixSet.map(gcMatrixSet => processMatrixSet(gcMatrixSet, entry))+ : [processMatrixSet(gcTileMatrixSet, entry)]wv.sources[entry.source] = {matrixSets: Object.fromEntries(new Map(wvMatrixSets))@@ -182,34 +314,29 @@ async function processEntry (entry) {async function processLayer (gcLayer, wvLayers, entry) {const ident = gcLayer['ows:Identifier']._text- if (mode === 'verbose') console.trace(`Processing layer ${ident}...`)- if (skip.includes(ident)) {- console.log(`${ident}: skipping`)- throw new SkipException(ident)+ if (skipSet.has(ident)) {+ if (mode === 'verbose') {+ console.log(`${ident}: skipping`)+ }+ return { skipped: true }}- wvLayers[ident] = {}- let wvLayer = wvLayers[ident]+ const wvLayer = {}wvLayer.id = identwvLayer.type = 'wmts'wvLayer.format = gcLayer.Format._text- const temporalForProjection = {}-- // Extract start and end dates- if ('Dimension' in gcLayer) {- const dimension = gcLayer.Dimension- if (dimension['ows:Identifier']._text === 'Time') {- try {- wvLayer = await processTemporalLayer(wvLayer, dimension.Value, entry.source, cacheMode)- } catch (e) {- console.error(e)- console.error(`${prog}: ERROR: [${ident}] Error processing time values.`)- }+ // Temporal dimension+ const dimension = gcLayer.Dimension+ if (dimension && dimension['ows:Identifier']?._text === 'Time') {+ try {+ await processTemporalLayer(wvLayer, dimension.Value, entry.source, cacheMode)+ } catch (e) {+ console.error(`${prog}: ERROR: [${ident}] Temporal processing failed: ${e}`)}}- // Extract matrix set+ // Matrix setconst matrixSetLink = gcLayer.TileMatrixSetLinkconst matrixSet = matrixSetLink.TileMatrixSet._text@@ -219,81 +346,54 @@ async function processLayer (gcLayer, wvLayers, entry) {}wvLayer.projections = {}- if (temporalForProjection[entry.projection]) {- wvLayer.projections[entry.projection] = {- ...projectionInfo,- ...temporalForProjection[entry.projection]- }- delete wvLayer.dateRanges- delete wvLayer.startDate- delete wvLayer.endDate- } else {- wvLayer.projections[entry.projection] = { ...projectionInfo }- }+ wvLayer.projections[entry.projection] = { ...projectionInfo }- if (Object.prototype.hasOwnProperty.call(matrixSetLink, 'TileMatrixSetLimits') && matrixSetLink.TileMatrixSetLimits !== null) {+ if (matrixSetLink.TileMatrixSetLimits) {const matrixSetLimits = matrixSetLink.TileMatrixSetLimits.TileMatrixLimits- const mappedSetLimits = []- for (const setLimit of matrixSetLimits) {- mappedSetLimits.push({- tileMatrix: setLimit.TileMatrix._text,- minTileRow: parseInt(setLimit.MinTileRow._text, 10),- maxTileRow: parseInt(setLimit.MaxTileRow._text, 10),- minTileCol: parseInt(setLimit.MinTileCol._text, 10),- maxTileCol: parseInt(setLimit.MaxTileCol._text, 10)- })- }- wvLayer.projections[entry.projection].matrixSetLimits = mappedSetLimits+ wvLayer.projections[entry.projection].matrixSetLimits = matrixSetLimits.map(setLimit => ({+ tileMatrix: setLimit.TileMatrix._text,+ minTileRow: parseInt(setLimit.MinTileRow._text, 10),+ maxTileRow: parseInt(setLimit.MaxTileRow._text, 10),+ minTileCol: parseInt(setLimit.MinTileCol._text, 10),+ maxTileCol: parseInt(setLimit.MaxTileCol._text, 10)+ }))}- // Vector data links- if ((Object.prototype.hasOwnProperty.call(gcLayer, 'ows:Metadata') && gcLayer['ows:Metadata'] !== null)) {- for (const item of gcLayer['ows:Metadata']) {- if (!(['xlink:role'] in item._attributes)) {- throw new Error('No xlink:role')- }- const schemaVersion = item._attributes['xlink:role']-- if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {- const vectorDataLink = item._attributes['xlink:href']- const vectorDataFile = path.basename(vectorDataLink)- const vectorDataId = path.parse(vectorDataFile).name- wvLayer.vectorData = {- id: vectorDataId- }- }- }- }-- if (('ows:Metadata' in gcLayer) && (gcLayer['ows:Metadata'] !== null)) {- if (('skipPalettes' in config) && (ident in config.skipPalettes)) {- console.warn(`${prog}: WARNING: Skipping palette for ${ident}`)+ // ows:Metadata (vector data, palettes, vector style)+ const metadataArr = gcLayer['ows:Metadata']+ if (metadataArr) {+ if (skipPalettesSet.has(ident)) {totalWarningCount+++ if (mode === 'verbose') {+ console.warn(`${prog}: WARNING: Skipping palette for ${ident}`)+ }} else {- for (const item of gcLayer['ows:Metadata']) {- if (!(['xlink:role'] in item._attributes)) {+ for (const item of metadataArr) {+ const role = item._attributes?.['xlink:role']+ if (!role) {throw new Error('No xlink:role')}- const schemaVersion = item._attributes['xlink:role']-- if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {- const colormapLink = item._attributes['xlink:href']- const colormapFile = path.basename(colormapLink)+ const href = item._attributes['xlink:href']+ if (!href) continue++ if (role === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {+ const vectorDataFile = path.basename(href)+ const vectorDataId = path.parse(vectorDataFile).name+ wvLayer.vectorData = { id: vectorDataId }+ } else if (role === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {+ const colormapFile = path.basename(href)const colormapId = path.parse(colormapFile).name- wvLayer.palette = {- id: colormapId- }- } else if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0') {- const vectorstyleLink = item._attributes['xlink:href']- const vectorstyleFile = path.basename(vectorstyleLink)+ wvLayer.palette = { id: colormapId }+ } else if (role === 'http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0') {+ const vectorstyleFile = path.basename(href)const vectorstyleId = path.parse(vectorstyleFile).name- wvLayer.vectorStyle = {- id: vectorstyleId- }+ wvLayer.vectorStyle = { id: vectorstyleId }}}}}++ wvLayers[ident] = wvLayer}function processMatrixSet (gcMatrixSet, entry) {@@ -302,9 +402,11 @@ function processMatrixSet (gcMatrixSet, entry) {const zoomLevels = tileMatrixArr.lengthconst resolutions = []const formattedTileMatrixArr = []- const maxResolution = entry.maxResolution++ let current = entry.maxResolutionfor (let zoom = 0; zoom < zoomLevels; zoom += 1) {- resolutions.push(maxResolution / (2 ** zoom))+ resolutions.push(current)+ current /= 2}for (const tileMatrix of tileMatrixArr) {@@ -316,7 +418,7 @@ function processMatrixSet (gcMatrixSet, entry) {const value = {id: ident,- maxResolution,+ maxResolution: entry.maxResolution,resolutions,tileSize: [parseInt(tileMatrixArr[0].TileWidth._text, 10),@@ -331,3 +433,10 @@ function processMatrixSet (gcMatrixSet, entry) {main().catch((err) => {console.error(err.stack)})++module.exports = {+ processEntry,+ processLayer,+ processMatrixSet,+ main+}