Interactive interface for browsing global, full-resolution satellite imagery
python conversion wip
zachtrice136e549cb284
diff --git a/package-lock.json b/package-lock.jsonindex 284855ce3..e3b958326 100644--- a/package-lock.json+++ b/package-lock.json@@ -124,6 +124,7 @@"react-refresh": "^0.14.0","react-test-renderer": "^16.12.0","redux-mock-store": "^1.5.3",+ "regex-parser": "^2.2.11","run-script-os": "^1.1.5","sass": "^1.49.7","sass-loader": "^10.1.0",@@ -18822,6 +18823,12 @@"node": ">=0.10.0"}},+ "node_modules/regex-parser": {+ "version": "2.2.11",+ "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz",+ "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==",+ "dev": true+ },"node_modules/regexp.prototype.flags": {"version": "1.4.3","resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",@@ -37529,6 +37536,12 @@"safe-regex": "^1.1.0"}},+ "regex-parser": {+ "version": "2.2.11",+ "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz",+ "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==",+ "dev": true+ },"regexp.prototype.flags": {"version": "1.4.3","resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",diff --git a/package.json b/package.jsonindex 236cfb56f..ca068a47f 100644--- a/package.json+++ b/package.json@@ -138,6 +138,7 @@"react-refresh": "^0.14.0","react-test-renderer": "^16.12.0","redux-mock-store": "^1.5.3",+ "regex-parser": "^2.2.11","run-script-os": "^1.1.5","sass": "^1.49.7","sass-loader": "^10.1.0",diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 552927c54..e604e4bcb 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -82,24 +82,24 @@ if [ -e "$BUILD_DIR/config.json" ] ; then`node $NODE_SCRIPTS_DIR/extractConfigFromWMTS.js \--config "$BUILD_DIR/config.json" \--inputDir "$BUILD_DIR/gc" \- --outputDir "$BUILD_DIR/_wmts"+ --outputDir "$BUILD_DIR/_wmts"`fi# # Run processVectorStyles.py and move vectorstyles where we want them-# if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then-# mkdir -p "$BUILD_DIR"/config/wv.json/vectorstyles-# "$PYTHON_SCRIPTS_DIR/processVectorStyles.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \-# "$BUILD_DIR/gc/vectorstyles" \-# "$BUILD_DIR/config/wv.json/vectorstyles"-# fi+if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then+ mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"+ `node $NODE_SCRIPTS_DIR/processVectorStyles.js \+ --inputDir "$BUILD_DIR/gc/vectorstyles" \+ --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`+fi-# # Run processVectorData.py and move vectordata where we want them-# if [ -e "$BUILD_DIR/gc/vectordata" ] ; then-# mkdir -p "$BUILD_DIR"/config/wv.json/vectordata-# "$PYTHON_SCRIPTS_DIR/processVectorData.py" "$OPT_DIR/$OPT_SUBDIR/config.json" \-# "$BUILD_DIR/gc/vectordata" \-# "$BUILD_DIR/config/wv.json/vectordata"-# fi+# Run processVectorData.py and move vectordata where we want them+if [ -e "$BUILD_DIR/gc/vectordata" ] ; then+ mkdir -p "$BUILD_DIR/config/wv.json/vectordata"+ `node $NODE_SCRIPTS_DIR/processVectorData.js \+ --inputDir "$BUILD_DIR/gc/vectordata" \+ --outputDir "$BUILD_DIR/config/wv.json/vectordata"`+fi# # Run processColormap.py and move colormaps where we want them# if [ -e "$BUILD_DIR/colormaps" ] ; thendiff --git a/tasks/node/extractConfigFromWMTS.js b/tasks/node/extractConfigFromWMTS.jsindex 1215656c8..52f4001f8 100755--- a/tasks/node/extractConfigFromWMTS.js+++ b/tasks/node/extractConfigFromWMTS.js@@ -2,6 +2,7 @@ const fs = require('fs')const path = require('path')const yargs = require('yargs')const console = require('console')+const processTemporalLayer = require('./processTemporalLayer')const prog = path.basename(__filename)@@ -33,9 +34,8 @@ if (!argv.config && !argv.inputDir && !argv.outputDir) {}const configFile = argv.config-const configData = fs.readFileSync(configFile)-const config = JSON.parse(configData)-// const inputDir = argv.inputDir+const config = JSON.parse(fs.readFileSync(configFile, 'utf-8'))+const inputDir = argv.inputDirconst outputDir = argv.outputDirif (!Object.prototype.hasOwnProperty.call(config, 'wv-options-wmts')) {@@ -48,15 +48,18 @@ if (!fs.existsSync(outputDir)) {const tolerant = config.tolerantconst entries = config['wv-options-wmts']-// let skip = []+const skip = config.skip || []-// if (config.skip) {-// skip = config.skip-// }+const totalLayerCount = 0+let totalWarningCount = 0+const totalErrorCount = 0-// const totalLayerCount = 0-// const totalWarningCount = 0-// const totalErrorCount = 0+class SkipException extends Error {+ constructor (message) {+ super(message)+ this.name = 'SkipException'+ }+}async function main () {for (entry of entries) {@@ -75,52 +78,234 @@ async function main () {}}-// async function processLayer (gcLayer, wvLayers, entry) {+function processLayer (gcLayer, wvLayers, entry) {+ const ident = gcLayer['ows:Identifier']+ if (skip.includes(ident)) {+ console.log(`${ident}: skipping`)+ throw new SkipException(ident)+ }-// }+ wvLayers[ident] = {}+ let wvLayer = wvLayers[ident]+ wvLayer.id = ident+ wvLayer.type = 'wmts'+ wvLayer.format = gcLayer.Format-async function processEntry (entry) {- const inputFile = path.join(inputDir, entry.from)- const gcId = path.basename(inputFile)+ const temporalForProjection = {}++ // Extract start and end dates+ if ('Dimension' in gcLayer) {+ const dimension = gcLayer.Dimension+ if (dimension['ows:Identifier'] === 'Time') {+ try {+ wvLayer = processTemporalLayer(wvLayer, dimension.Value)+ } catch (e) {+ console.error(e)+ console.error(`${prog}: ERROR: [${ident}] Error processing time values.`)+ }+ }+ }++ // Extract matrix set+ const matrixSetLink = gcLayer.TileMatrixSetLink+ const matrixSet = matrixSetLink.TileMatrixSet++ const projectionInfo = {+ source: entry.source,+ matrixSet+ }+ wvLayer.projections = {}+ if (temporalForProjection[entry.projection]) {+ wvLayer.projections[entry.projection] = {+ ...projectionInfo,+ ...temporalForProjection[entry.projection]+ }+ delete wvLayer.dateRanges+ delete wvLayer.startDate+ delete wvLayer.endDate+ } else {+ wvLayer.projections[entry.projection] = { ...projectionInfo }+ }++ if (Object.prototype.hasOwnProperty.call(matrixSetLink, 'TileMatrixSetLimits') && matrixSetLink.TileMatrixSetLimits !== null) {+ const matrixSetLimits = matrixSetLink.TileMatrixSetLimits.TileMatrixLimits+ const mappedSetLimits = []+ for (const setLimit of matrixSetLimits) {+ mappedSetLimits.push({+ tileMatrix: setLimit.TileMatrix,+ minTileRow: parseInt(setLimit.MinTileRow, 10),+ maxTileRow: parseInt(setLimit.MaxTileRow, 10),+ minTileCol: parseInt(setLimit.MinTileCol, 10),+ maxTileCol: parseInt(setLimit.MaxTileCol, 10)+ })+ }+ wvLayer.projections[entry.projection].matrixSetLimits = mappedSetLimits+ }++ // Vector data links+ if ((Object.prototype.hasOwnProperty.call(gcLayer, 'ows:Metadata') && gcLayer['ows:Metadata'] !== null)) {+ for (const item of gcLayer['ows:Metadata']) {+ if (!Object.prototype.hasOwnProperty.call(item, '@xlink:role')) {+ throw new Error('No xlink:role')+ }+ const schemaVersion = item['@xlink:role']++ if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {+ const vectorDataLink = item['@xlink:href']+ const vectorDataFile = path.basename(vectorDataLink)+ const vectorDataId = path.splitext(vectorDataFile)[0]+ wvLayer.vectorData = {+ id: vectorDataId+ }+ }+ }+ }++ if (('ows:Metadata' in gcLayer) && (gcLayer['ows:Metadata'] !== null)) {+ if (('skipPalettes' in config) && (ident in config.skipPalettes)) {+ console.warn(`${prog}: WARNING: Skipping palette for ${ident}`)+ totalWarningCount+++ } else {+ for (const item of gcLayer['ows:Metadata']) {+ if (!('@xlink:role' in item)) {+ throw new Error('No xlink:role')+ }+ const schemaVersion = item['@xlink:role']++ if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {+ const colormapLink = item['@xlink:href']+ const colormapFile = path.basename(colormapLink)+ const colormapId = path.splitext(colormapFile)[0]+ wvLayer.palette = {+ id: colormapId+ }+ } else if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0') {+ const vectorstyleLink = item['@xlink:href']+ const vectorstyleFile = path.basename(vectorstyleLink)+ const vectorstyleId = path.splitext(vectorstyleFile)[0]+ wvLayer.vectorStyle = {+ id: vectorstyleId+ }+ }+ }+ }+ }+}++async function processEntry (entry) {+ let layerCount = 0+ let warningCount = 0+ let errorCount = 0const wv = {layers: {},sources: {}}- // const wvMatrixSets = {}+ const wvMatrixSets = {}+ wv.sources[entry.source] = {+ matrixSets: wvMatrixSets+ }+ const inputFile = path.join(inputDir, entry.from)+ const gcId = path.basename(inputFile)let gc-try {- fs.readFileSync(inputFile)- const gcXML = fs.readFileSync(inputFile, { encoding: 'utf-8' })- gc = convert.xml2json(gcXML, { compact: true, spaces: 2 })- gc = JSON.parse(gc)- } catch (error) {+ const xml = await fs.promises.readFile(inputFile, 'utf8')+ gc = xmltodict.parse(xml)+ } catch (e) {if (tolerant) {+ process.stderr.write(`${prog}: WARN: [${inputFile}] Unable to get GC: ${e}\n`)warningCount += 1- console.warn(`${prog}: WARN: ${inputFile} Unable to get GC: ${error}`)} else {+ process.stderr.write(`${prog}: ERROR: [${inputFile}] Unable to get GC: ${e}\n`)errorCount += 1- console.error(`${prog}: ERROR: ${inputFile} Unable to get GC: ${error}`)}- return { layerCount, warningCount, errorCount }+ return [errorCount, warningCount, layerCount]}- // TODO: errors can go uncaught here- gcContents = gc.Capabilities.Contents- wvLayers = wv.layers+ const gcContents = gc.Capabilities.Contents+ const wvLayers = wv.layersif (!gcContents || !gcContents.Layer) {errorCount += 1- console.error(`${prog}: ERROR: ${gcId} No Layers`)- return { layerCount, warningCount, errorCount }+ process.stderr.write(`${prog}: ERROR: [${gcId}] No layers\n`)+ return [errorCount, warningCount, layerCount]+ }++ if (typeof gc.Capabilities.Contents.Layer === 'object') {+ const gcLayer = gc.Capabilities.Contents.Layer+ const ident = gcLayer['ows:Identifier']+ try {+ layerCount += 1+ processLayer(gcLayer, wvLayers, entry)+ } catch (error) {+ if (error instanceof SkipException) {+ warningCount += 1+ process.stderr.write(`${prog}: WARNING: [${ident}] Skipping\n`)+ } else {+ errorCount += 1+ console.error(e.stack)+ process.stderr.write(`${prog}: ERROR: [${gcId}:${ident}] ${e}\n`)+ }+ }+ } else {+ for (const gcLayer of gcContents.Layer) {+ // const ident = gcLayer['ows:Identifier']+ try {+ layerCount += 1+ processLayer(gcLayer, wvLayers, entry)+ } catch (error) {+ if (se instanceof SkipException) {+ warningCount += 1+ process.stderr.write(`${prog}: WARNING: [${id}] Skipping\n`)+ } else {+ errorCount += 1+ console.error(e.stack)+ }+ }+ }+ }++ processMatrixSet(gcMatrixSet)++ if (typeof gcContents.TileMatrixSet === 'object') {+ processMatrixSet(gcContents.TileMatrixSet)+ } else {+ gcContents.TileMatrixSet.forEach(gcMatrixSet => {+ processMatrixSet(gcMatrixSet)+ })}}-// async function processMatrixSet (gcMatricSet) {+function processMatrixSet (gcMatrixSet) {+ const tileMatrixArr = gcMatrixSet.TileMatrix+ const ident = gcMatrixSet['ows:Identifier']+ const zoomLevels = tileMatrixArr.length+ const resolutions = []+ const formattedTileMatrixArr = []+ const maxResolution = entry.maxResolution+ for (let zoom = 0; zoom < zoomLevels; zoom += 1) {+ resolutions.push(maxResolution / (2 ** zoom))+ }++ for (const tileMatrix of tileMatrixArr) {+ formattedTileMatrixArr.push({+ matrixWidth: parseInt(tileMatrix.MatrixWidth, 10),+ matrixHeight: parseInt(tileMatrix.MatrixHeight, 10)+ })+ }-// }+ wvMatrixSets[ident] = {+ id: ident,+ maxResolution,+ resolutions,+ tileSize: [+ parseInt(tileMatrixArr[0].TileWidth, 10),+ parseInt(tileMatrixArr[0].TileHeight, 10)+ ],+ tileMatrices: formattedTileMatrixArr+ }+}main().catch((err) => {console.error(err.stack)diff --git a/tasks/node/processTemporalLayer.js b/tasks/node/processTemporalLayer.jsnew file mode 100644index 000000000..2e9fcb87d--- /dev/null+++ b/tasks/node/processTemporalLayer.js@@ -0,0 +1,97 @@+const moment = require('moment')+const re = require('regex-parser')++function toList (val) {+ return val instanceof Array ? val : [val]+}++module.exports = function processTemporal (wvLayer, value) {+ const dateFormat = 'YYYY-MM-DD'+ const timeFormat = 'HH:mm:ss'+ const dateTimeFormat = 'YYYY-MM-DD HH:mm:ss'+ try {+ const ranges = toList(value)+ if (ranges[0].includes('T')) {+ wvLayer.period = 'subdaily'+ } else {+ if (ranges[0].endsWith('Y')) {+ wvLayer.period = 'yearly'+ } else if (ranges[0].endsWith('M')) {+ wvLayer.period = 'monthly'+ } else {+ wvLayer.period = 'daily'+ }+ }+ let startDate = moment.min()+ let endDate = moment.max()+ const dateRangeStart = []+ const dateRangeEnd = []+ const rangeInterval = []+ for (const range of ranges) {+ const [start, end, interval] = range.split('/')+ if (+ wvLayer.period === 'daily' ||+ wvLayer.period === 'monthly' ||+ wvLayer.period === 'yearly'+ ) {+ startDate = moment.min(startDate, moment(start, dateFormat))+ endDate = moment.max(endDate, moment(end, dateFormat))+ if (startDate) {+ const startDateParse = moment(start, dateFormat)+ dateRangeStart.push(+ `${startDateParse.format(dateFormat)}T${startDateParse.format(+ timeFormat+ )}Z`+ )+ }+ if (endDate) {+ const endDateParse = moment(end, dateFormat)+ dateRangeEnd.push(+ `${endDateParse.format(dateFormat)}T${endDateParse.format(+ timeFormat+ )}Z`+ )+ }+ if (interval !== 'P1D') {+ endDate = moment(endDate).add(moment.duration(interval)).toDate()+ }+ rangeInterval.push(re.parse(interval).match)+ } else {+ // Subdaily Layers+ const startTime = start.replace('T', ' ').replace('Z', '')+ const endTime = end.replace('T', ' ').replace('Z', '')+ startDate = moment.min(startDate, moment(startTime, dateTimeFormat))+ endDate = moment.max(endDate, moment(endTime, dateTimeFormat))++ if (startDate) {+ const startTimeParse = moment(startTime, dateTimeFormat)+ dateRangeStart.push(startTimeParse.format(`${dateFormat}T${timeFormat}Z`))+ }+ if (endDate) {+ const endTimeParse = moment(endTime, dateTimeFormat)+ dateRangeEnd.push(endTimeParse.format(`${dateFormat}T${timeFormat}Z`))+ }++ rangeInterval.push(re.search(/\d+/, interval).group())+ }++ wvLayer.startDate = moment(startDate).format(`${dateFormat}T${timeFormat}Z`)+ if (!endDate.isSame(moment.min())) {+ wvLayer.endDate = moment(endDate).format(`${dateFormat}T${timeFormat}Z`)+ }+ if (dateRangeStart.length && dateRangeEnd.length) {+ wvLayer.dateRanges = dateRangeStart.map((s, i) => ({+ startDate: s,+ endDate: dateRangeEnd[i],+ dateInterval: rangeInterval[i]+ }))+ }+ }+ } catch (e) {+ if (e instanceof ValueError) {+ throw new Error(`Invalid time: ${range}`)+ }+ throw new Error(`Error processing temporal layer: ${e}`)+ }+ return wvLayer+}diff --git a/tasks/node/processVectorData.js b/tasks/node/processVectorData.jsnew file mode 100644index 000000000..8b80d3b77--- /dev/null+++ b/tasks/node/processVectorData.js@@ -0,0 +1,76 @@+const fs = require('fs')+const path = require('path')+const { promisify } = require('util')++const readFile = promisify(fs.readFile)+const writeFile = promisify(fs.writeFile)+const copyFile = promisify(fs.copyFile)++const prog = path.basename(__filename)++const options = yargs+ .usage('Usage: $0 [options]')+ .option('inputDir', {+ demandOption: true,+ alias: 'i',+ type: 'string',+ description: 'getcapabilities input directory'+ })+ .option('outputDir', {+ demandOption: true,+ alias: 'o',+ type: 'string',+ description: 'wmts output directory'+ })+ .epilog('Extracts vector data information from GetCapabilities')++const { argv } = options+if (!argv.config && !argv.inputDir && !argv.outputDir) {+ throw new Error('Invalid number of arguments')+}++const inputDir = argv.inputDir+const outputDir = argv.outputDir++async function main () {+ let fileCount = 0+ let errorCount = 0++ for (const files of inputDir) {+ for (const file of fs.readdirSync(files)) {+ try {+ fileCount += 1+ await copyFileAsync(file)+ } catch (error) {+ errorCount += 1+ console.error(`${prog}: ERROR: [${file}] ${e}\n.`)+ }+ }+ }++ console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)++ if (errorCount > 0) {+ throw new Error(`${prog}: Error: ${errorCount.length} errors occured`)+ }+}++async function copyFileAsync (file) {+ if (inputFile.endsWith('.json')) {+ const responseData = {}+ const vectorLayerFilename = file+ const vectorLayerId = vectorLayerFilename.split('.json', 1)[0]+ responseData.vectorData = {}+ responseData.vectorData[vectorLayerId] = {}+ const initialData = JSON.parse(await readFile(inputFile, 'utf-8'))+ for (const i in initialData) {+ responseData.vectorData[vectorLayerId][i] = initialData[i]+ }+ await writeFile(inputFile, JSON.stringify(responseData, null, 2), 'utf-8')+ await copyFile(inputFile, outputDir)+ }+}++main().catch((err) => {+ console.error(err.stack)+})diff --git a/tasks/node/processVectorStyles.js b/tasks/node/processVectorStyles.jsnew file mode 100644index 000000000..3664084c0--- /dev/null+++ b/tasks/node/processVectorStyles.js@@ -0,0 +1,76 @@+const fs = require('fs')+const path = require('path')+const { promisify } = require('util')++const readFile = promisify(fs.readFile)+const writeFile = promisify(fs.writeFile)+const copyFile = promisify(fs.copyFile)++const prog = path.basename(__filename)++const options = yargs+ .usage('Usage: $0 [options]')+ .option('inputDir', {+ demandOption: true,+ alias: 'i',+ type: 'string',+ description: 'getcapabilities input directory'+ })+ .option('outputDir', {+ demandOption: true,+ alias: 'o',+ type: 'string',+ description: 'wmts output directory'+ })+ .epilog('Extracts vector style information from GetCapabilities')++const { argv } = options+if (!argv.config && !argv.inputDir && !argv.outputDir) {+ throw new Error('Invalid number of arguments')+}++const inputDir = argv.inputDir+const outputDir = argv.outputDir++async function main () {+ let fileCount = 0+ let errorCount = 0++ for (const files of inputDir) {+ for (const file of fs.readdirSync(files)) {+ try {+ fileCount += 1+ await copyFileAsync(file)+ } catch (error) {+ errorCount += 1+ console.error(`${prog}: ERROR: [${file}] ${e}\n.`)+ }+ }+ }++ console.error(`${prog}: ${errorCount} error(s), ${fileCount} file(s)`)++ if (errorCount > 0) {+ throw new Error(`${prog}: Error: ${errorCount.length} errors occured`)+ }+}++async function copyFileAsync (file) {+ if (inputFile.endsWith('.json')) {+ const responseData = {}+ const vectorLayerFilename = file+ const vectorLayerId = vectorLayerFilename.split('.json', 1)[0]+ responseData.vectorStyles = {}+ responseData.vectorStyles[vectorLayerId] = {}+ const initialData = JSON.parse(await readFile(inputFile, 'utf-8'))+ for (const i in initialData) {+ responseData.vectorStyles[vectorLayerId][i] = initialData[i]+ }+ await writeFile(inputFile, JSON.stringify(responseData, null, 2), 'utf-8')+ await copyFile(inputFile, outputDir)+ }+}++main().catch((err) => {+ console.error(err.stack)+})