Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

Fix most issues with extractConfigFromWMTS and processTemporalLayer

ZachTRicede6fc92d198c
diff --git a/package-lock.json b/package-lock.json
index 678641023..86c6c6ac2 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -124,7 +124,6 @@
"react-refresh": "^0.14.0",
"react-test-renderer": "^16.12.0",
"redux-mock-store": "^1.5.3",
- "regex-parser": "^2.2.11",
"run-script-os": "^1.1.5",
"sass": "^1.49.7",
"sass-loader": "^10.1.0",
@@ -18824,12 +18823,6 @@
"node": ">=0.10.0"
}
},
- "node_modules/regex-parser": {
- "version": "2.2.11",
- "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz",
- "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==",
- "dev": true
- },
"node_modules/regexp.prototype.flags": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",
@@ -37559,12 +37552,6 @@
"safe-regex": "^1.1.0"
}
},
- "regex-parser": {
- "version": "2.2.11",
- "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz",
- "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==",
- "dev": true
- },
"regexp.prototype.flags": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",
diff --git a/package.json b/package.json
index 3117bbeeb..676dfb06f 100644
--- a/package.json
+++ b/package.json
@@ -135,7 +135,6 @@
"react-refresh": "^0.14.0",
"react-test-renderer": "^16.12.0",
"redux-mock-store": "^1.5.3",
- "regex-parser": "^2.2.11",
"run-script-os": "^1.1.5",
"sass": "^1.49.7",
"sass-loader": "^10.1.0",
diff --git a/tasks/build/extractConfigFromWMTS.js b/tasks/build/extractConfigFromWMTS.js
index 177efdca4..0bcc3ddb6 100644
--- a/tasks/build/extractConfigFromWMTS.js
+++ b/tasks/build/extractConfigFromWMTS.js
@@ -3,7 +3,7 @@ const path = require('path')
const yargs = require('yargs')
const console = require('console')
const convert = require('xml-js')
-const processTemporalLayer = require('./processTemporalLayer')
+const { processTemporalLayer } = require('./processTemporalLayer')
const prog = path.basename(__filename)
@@ -56,6 +56,11 @@ let totalLayerCount = 0
let totalWarningCount = 0
let totalErrorCount = 0
+const wv = {
+ layers: {},
+ sources: {}
+}
+
class SkipException extends Error {
constructor (message) {
super(message)
@@ -74,6 +79,13 @@ async function main () {
const { errorCount, warningCount, layerCount } = await processEntry(entry)
console.warn(`${prog}: ${errorCount} errors, ${warningCount} warnings, ${layerCount} layers for ${entry.source}`)
+ const outputFile = path.join(outputDir, entry.to)
+ fs.writeFile(outputFile, JSON.stringify(wv, null, 2), 'utf-8', err => {
+ if (err) {
+ console.error(err)
+ }
+ })
+
totalErrorCount += errorCount
totalWarningCount += warningCount
totalLayerCount += layerCount
@@ -86,8 +98,88 @@ async function main () {
}
}
-function processLayer (gcLayer, wvLayers, entry) {
- const ident = gcLayer['ows:Identifier']
+async function processEntry (entry) {
+ let layerCount = 0
+ let warningCount = 0
+ let errorCount = 0
+
+ wv.sources[entry.source] = {
+ matrixSets: wvMatrixSets
+ }
+
+ const inputFile = path.join(inputDir, entry.from)
+ const gcId = path.basename(inputFile)
+ let gc
+ try {
+ const xml = await fs.promises.readFile(inputFile, 'utf8')
+ gc = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))
+ } catch (e) {
+ if (tolerant) {
+ process.stderr.write(`${prog}: WARN: [${inputFile}] Unable to get GC: ${e}\n`)
+ warningCount += 1
+ } else {
+ process.stderr.write(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)
+ errorCount += 1
+ }
+ return { errorCount, warningCount, layerCount }
+ }
+
+ const gcContents = gc.Capabilities.Contents
+ const wvLayers = wv.layers
+
+ if (!gcContents || !gcContents.Layer) {
+ errorCount += 1
+ process.stderr.write(`${prog}: ERROR: [${gcId}] No layers\n`)
+ return { errorCount, warningCount, layerCount }
+ }
+
+ // if (typeof gc.Capabilities.Contents.Layer === 'object') {
+ // const gcLayer = gc.Capabilities.Contents.Layer
+ // const ident = gcLayer['ows:Identifier']
+ // try {
+ // layerCount += 1
+ // processLayer(gcLayer, wvLayers, entry)
+ // } catch (error) {
+ // if (error instanceof SkipException) {
+ // warningCount += 1
+ // process.stderr.write(`${prog}: WARNING: [${ident}] Skipping\n`)
+ // } else {
+ // errorCount += 1
+ // console.error(error.stack)
+ // process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)
+ // }
+ // }
+ // } else {
+ for (const gcLayer of gcContents.Layer) {
+ try {
+ layerCount += 1
+ processLayer(gcLayer, wvLayers, entry)
+ } catch (error) {
+ if (error instanceof SkipException) {
+ warningCount += 1
+ process.stderr.write(`${prog}: WARNING: [${id}] Skipping\n`)
+ } else {
+ errorCount += 1
+ console.error(error.stack)
+ process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)
+ }
+ }
+ }
+ // }
+
+ if (gcContents.TileMatrixSet === 'Object') {
+ processMatrixSet(gcContents.TileMatrixSet)
+ } else {
+ gcContents.TileMatrixSet.forEach(gcMatrixSet => {
+ processMatrixSet(gcMatrixSet)
+ })
+ }
+
+ return { errorCount, warningCount, layerCount }
+}
+
+async function processLayer (gcLayer, wvLayers, entry) {
+ const ident = gcLayer['ows:Identifier']._text
if (skip.includes(ident)) {
console.log(`${ident}: skipping`)
throw new SkipException(ident)
@@ -97,16 +189,16 @@ function processLayer (gcLayer, wvLayers, entry) {
let wvLayer = wvLayers[ident]
wvLayer.id = ident
wvLayer.type = 'wmts'
- wvLayer.format = gcLayer.Format
+ wvLayer.format = gcLayer.Format._text
const temporalForProjection = {}
// Extract start and end dates
if ('Dimension' in gcLayer) {
const dimension = gcLayer.Dimension
- if (dimension['ows:Identifier'] === 'Time') {
+ if (dimension['ows:Identifier']._text === 'Time') {
try {
- wvLayer = processTemporalLayer(wvLayer, dimension.Value)
+ wvLayer = await processTemporalLayer(wvLayer, dimension.Value)
} catch (e) {
console.error(e)
console.error(`${prog}: ERROR: [${ident}] Error processing time values.`)
@@ -116,7 +208,7 @@ function processLayer (gcLayer, wvLayers, entry) {
// Extract matrix set
const matrixSetLink = gcLayer.TileMatrixSetLink
- const matrixSet = matrixSetLink.TileMatrixSet
+ const matrixSet = matrixSetLink.TileMatrixSet._text
const projectionInfo = {
source: entry.source,
@@ -141,7 +233,7 @@ function processLayer (gcLayer, wvLayers, entry) {
const mappedSetLimits = []
for (const setLimit of matrixSetLimits) {
mappedSetLimits.push({
- tileMatrix: setLimit.TileMatrix,
+ tileMatrix: setLimit.TileMatrix._text,
minTileRow: parseInt(setLimit.MinTileRow, 10),
maxTileRow: parseInt(setLimit.MaxTileRow, 10),
minTileCol: parseInt(setLimit.MinTileCol, 10),
@@ -201,92 +293,9 @@ function processLayer (gcLayer, wvLayers, entry) {
}
}
-async function processEntry (entry) {
- let layerCount = 0
- let warningCount = 0
- let errorCount = 0
- const wv = {
- layers: {},
- sources: {}
- }
-
- wv.sources[entry.source] = {
- matrixSets: wvMatrixSets
- }
-
- const inputFile = path.join(inputDir, entry.from)
- const gcId = path.basename(inputFile)
- let gc
- try {
- const xml = await fs.promises.readFile(inputFile, 'utf8')
- gc = JSON.parse(convert.xml2json(xml, { compact: true, spaces: 2 }))
- } catch (e) {
- if (tolerant) {
- process.stderr.write(`${prog}: WARN: [${inputFile}] Unable to get GC: ${e}\n`)
- warningCount += 1
- } else {
- process.stderr.write(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)
- errorCount += 1
- }
- return { errorCount, warningCount, layerCount }
- }
-
- const gcContents = gc.Capabilities.Contents
- const wvLayers = wv.layers
-
- if (!gcContents || !gcContents.Layer) {
- errorCount += 1
- process.stderr.write(`${prog}: ERROR: [${gcId}] No layers\n`)
- return { errorCount, warningCount, layerCount }
- }
-
- // if (typeof gc.Capabilities.Contents.Layer === 'object') {
- // const gcLayer = gc.Capabilities.Contents.Layer
- // const ident = gcLayer['ows:Identifier']
- // try {
- // layerCount += 1
- // processLayer(gcLayer, wvLayers, entry)
- // } catch (error) {
- // if (error instanceof SkipException) {
- // warningCount += 1
- // process.stderr.write(`${prog}: WARNING: [${ident}] Skipping\n`)
- // } else {
- // errorCount += 1
- // console.error(error.stack)
- // process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)
- // }
- // }
- // } else {
- for (const gcLayer of gcContents.Layer) {
- try {
- layerCount += 1
- processLayer(gcLayer, wvLayers, entry)
- } catch (error) {
- if (error instanceof SkipException) {
- warningCount += 1
- process.stderr.write(`${prog}: WARNING: [${id}] Skipping\n`)
- } else {
- errorCount += 1
- console.error(error.stack)
- process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)
- }
- }
- }
- // }
-
- if (gcContents.TileMatrixSet === 'Object') {
- processMatrixSet(gcContents.TileMatrixSet)
- } else {
- gcContents.TileMatrixSet.forEach(gcMatrixSet => {
- processMatrixSet(gcMatrixSet)
- })
- }
- return { errorCount, warningCount, layerCount }
-}
-
function processMatrixSet (gcMatrixSet) {
const tileMatrixArr = gcMatrixSet.TileMatrix
- const ident = gcMatrixSet['ows:Identifier']
+ const ident = gcMatrixSet['ows:Identifier']._text
const zoomLevels = tileMatrixArr.length
const resolutions = []
const formattedTileMatrixArr = []
diff --git a/tasks/build/mergeConfigWithWMTS.js b/tasks/build/mergeConfigWithWMTS.js
index 4bb5f8fd5..935e53c66 100644
--- a/tasks/build/mergeConfigWithWMTS.js
+++ b/tasks/build/mergeConfigWithWMTS.js
@@ -36,27 +36,37 @@ const outputData = JSON.parse(fs.readFileSync(outputFile, 'utf-8'))
let fileCount = 0
async function main () {
- fs.readdirSync(inputDir).forEach(file => {
- try {
- if (!file.endsWith('.json')) return
- fileCount += 1
- const data = JSON.parse(fs.readFileSync(path.join(inputDir, file), 'utf-8'))
- newConf.layers = wmtsDictMerge(newConf.layers, data, outputData)
- newConf.sources = dictMerge(newConf.sources, data.sources, outputData.sources)
- } catch (e) {
- throw new Error(`ERROR: ${path.join(inputDir, file)}: ${e.message}`)
- }
- })
-
- newConf = dictMerge(newConf, outputData)
+ await processFiles()
+ newConf = await dictMerge(newConf, outputData)
const jsonOptions = {}
jsonOptions.indent = 2
jsonOptions.separators = [',', ': ']
- fs.writeFileSync(outputFile, JSON.stringify(newConf, jsonOptions))
+ await fs.writeFileSync(outputFile, JSON.stringify(newConf, jsonOptions))
+
+ console.warn(`${prog}: ${fileCount} file(s) merged into ${path.basename(outputFile)}`)
+}
- console.log(`${prog}: ${fileCount} file(s) merged into ${path.basename(outputFile)}`)
+async function processFiles () {
+ try {
+ console.warn(inputDir)
+ const files = await fs.readdirSync(inputDir)
+ for (const file of files) {
+ if (!file.endsWith('.json')) continue
+ fileCount += 1
+ try {
+ const data = await fs.readFile(path.join(inputDir, file), 'utf-8')
+ const jsonData = JSON.parse(data)
+ newConf.layers = wmtsDictMerge(newConf.layers, jsonData, outputData)
+ newConf.sources = dictMerge(newConf.sources, jsonData.sources, outputData.sources)
+ } catch (error) {
+ throw new Error(`ERROR: ${path.join(inputDir, file)}: ${error.message}`)
+ }
+ }
+ } catch (error) {
+ console.error(error)
+ }
}
function wmtsDictMerge (target, obj, conf) {
diff --git a/tasks/build/processTemporalLayer.js b/tasks/build/processTemporalLayer.js
index 6afb26307..b441d93dc 100644
--- a/tasks/build/processTemporalLayer.js
+++ b/tasks/build/processTemporalLayer.js
@@ -1,22 +1,29 @@
const moment = require('moment')
-const re = require('regex-parser')
function toList (val) {
return val instanceof Array ? val : [val]
}
-function processTemporal (wvLayer, value) {
+class ValueError extends Error {
+ constructor (message) {
+ super(message)
+ this.name = 'ValueError'
+ }
+}
+
+async function processTemporalLayer (wvLayer, value) {
const dateFormat = 'YYYY-MM-DD'
const timeFormat = 'HH:mm:ss'
const dateTimeFormat = 'YYYY-MM-DD HH:mm:ss'
try {
const ranges = toList(value)
- if (ranges[0].includes('T')) {
+ if (ranges && ranges[0] && ranges[0]._text && ranges[0]._text.includes('T')) {
+ console.warn(ranges[0]._text)
wvLayer.period = 'subdaily'
} else {
- if (ranges[0].endsWith('Y')) {
+ if (ranges && ranges[0] && ranges[0]._text && ranges[0]._text.endsWith('Y')) {
wvLayer.period = 'yearly'
- } else if (ranges[0].endsWith('M')) {
+ } else if (ranges && ranges[0] && ranges[0]._text && ranges[0]._text.endsWith('M')) {
wvLayer.period = 'monthly'
} else {
wvLayer.period = 'daily'
@@ -28,7 +35,7 @@ function processTemporal (wvLayer, value) {
const dateRangeEnd = []
const rangeInterval = []
for (const range of ranges) {
- const [start, end, interval] = range.split('/')
+ const [start, end, interval] = range._text.split('/')
if (
wvLayer.period === 'daily' ||
wvLayer.period === 'monthly' ||
@@ -53,9 +60,11 @@ function processTemporal (wvLayer, value) {
)
}
if (interval !== 'P1D') {
- endDate = moment(endDate).add(moment.duration(interval)).toDate()
+ endDate = moment(endDate).add(moment.duration(interval))
}
- rangeInterval.push(re.parse(interval).match)
+ const regex = new RegExp(/\d+/g)
+ const match = regex.exec(interval)
+ rangeInterval.push(match)
} else {
// Subdaily Layers
const startTime = start.replace('T', ' ').replace('Z', '')
@@ -72,7 +81,7 @@ function processTemporal (wvLayer, value) {
dateRangeEnd.push(endTimeParse.format(`${dateFormat}T${timeFormat}Z`))
}
- rangeInterval.push(re.search(/\d+/, interval).group())
+ rangeInterval.push(interval.match(/\d+/)[0])
}
wvLayer.startDate = moment(startDate).format(`${dateFormat}T${timeFormat}Z`)
@@ -97,5 +106,5 @@ function processTemporal (wvLayer, value) {
}
module.exports = {
- processTemporal
+ processTemporalLayer
}
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 3564cb37a..f7eea6d02 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -47,25 +47,25 @@ fi
mkdir -p "$DEST_DIR/config"
mkdir -p "$BUILD_DIR/colormaps"
-# # If $FETCH_GC is set, make various API requests
-# if [ "$FETCH_GC" ] ; then
-# # TODO: Fix axios fetch timeouts
-# # Fetch GC files and create colormaps, vectordata and vectorstyle files
-# rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
-# rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
-# `node $SCRIPTS_DIR/getCapabilities.js \
-# --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
-# --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`
+# If $FETCH_GC is set, make various API requests
+if [ "$FETCH_GC" ] ; then
+ # TODO: Fix axios fetch timeouts
+ # Fetch GC files and create colormaps, vectordata and vectorstyle files
+ rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
+ rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
+ `node $SCRIPTS_DIR/getCapabilities.js \
+ --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
+ --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`
# TODO: Optimize performance of this script
# Get metadata for files in layerOrder.json and combine this data into 1 file
- # rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
- # mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
- # `node $SCRIPTS_DIR/getVisMetadata.js \
- # --features "$BUILD_DIR/features.json" \
- # --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
- # --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`
-# fi
+ rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
+ mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
+ `node $SCRIPTS_DIR/getVisMetadata.js \
+ --features "$BUILD_DIR/features.json" \
+ --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
+ --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`
+fi
# Validate layers in wv.json with a JSON schema
`node $SCRIPTS_DIR/validateConfigs.js \
@@ -76,10 +76,7 @@ if [ -e "$BUILD_DIR/features.json" ] ; then
cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
fi
-# TODO: Fix sometimes the following error is thrown:
-# TypeError: Converting circular structure to JSON
-# extractConfigFromWMTS.js:85:11
-# this is coming from an issue in getCapabilities.js
+# TODO: Fix _wmts files containing all projections
# Run extractConfigFromWMTS.js script with config.json
if [ -e "$BUILD_DIR/config.json" ] ; then
`node $SCRIPTS_DIR/extractConfigFromWMTS.js \
@@ -146,9 +143,9 @@ for config in $configs; do
done
# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
-# `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
-# --inputDir "$BUILD_DIR/_wmts" \
-# --outputDir "$DEST_DIR/config/wv.json"`
+`node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
+ --inputDir "$BUILD_DIR/_wmts" \
+ --outputFile "$DEST_DIR/config/wv.json"`
# # Copy brand files from build to dest
# cp -r "$BUILD_DIR/brand" "$DEST_DIR"