Interactive interface for browsing global, full-resolution satellite imagery
convert getCapabilities + cleanup bamboo & ci/cd scripts
ZachTRice9d05319f24dd
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.ymlindex 2d48f5748..b9a94baf3 100644--- a/.github/workflows/ci-cd.yml+++ b/.github/workflows/ci-cd.yml@@ -12,19 +12,14 @@ jobs:fail-fast: falsematrix:os: [macOS-latest, ubuntu-latest, windows-latest]- node: [18]+ node: [16]steps:- name: Checkout the code- uses: actions/checkout@v2-- - name: Set up Python- uses: actions/setup-python@v2- with:- python-version: 3.9+ uses: actions/checkout@v3- name: Setup Node.js environment- uses: actions/[email protected]+ uses: actions/setup-node@v3with:node-version: ${{matrix.node}}diff --git a/.github/workflows/layer-pr-template.yml b/.github/workflows/layer-pr-template.ymlindex 5025da632..2c3c71e84 100644--- a/.github/workflows/layer-pr-template.yml+++ b/.github/workflows/layer-pr-template.yml@@ -11,7 +11,7 @@ jobs:name: 'Create PR with Layers Template'runs-on: ubuntu-lateststeps:- - uses: actions/checkout@v2+ - uses: actions/checkout@v3- name: pull-requestuses: repo-sync/pull-request@v2with:diff --git a/legacy/tasks/buildAll.sh b/legacy/tasks/buildAll.shdeleted file mode 100755index 965becf60..000000000--- a/legacy/tasks/buildAll.sh+++ /dev/null@@ -1,19 +0,0 @@-#!/bin/bash-BASE=$(dirname "$0")/..--# Bash commands to run in Bamboo to prepare an rpm for deployment--# Overwrite the placeholder config in the RPM directory with the actual-# bit.ly configuration-cp bitly.json rpm/bitly.json-# Bamboo automatically sets NODE_ENV to production. Override this so that-# devDependencies are installed.--npm ci --production=false --ignore-scripts-node node_modules/.bin/patch-package --allow-root-$BASE/tasks/pythonInstall.sh linux-pipenv run bash $BASE/tasks/distExplict.sh-node $BASE/rpm/buildRPM.js--# This is where bamboo expects to find artifacts-mv ~/rpmbuild build/rpm\ No newline at end of filediff --git a/package-lock.json b/package-lock.jsonindex 20c87478f..cb8003c3a 100644--- a/package-lock.json+++ b/package-lock.json@@ -82,6 +82,7 @@"@pmmmwh/react-refresh-webpack-plugin": "^0.5.7","@webpack-cli/serve": "^1.7.0","autoprefixer": "^10.4.12",+ "axios": "^1.2.1","babel-loader": "^8.1.0","browserstack-capabilities": "^0.7.0","browserstack-local": "^1.4.4",@@ -143,6 +144,7 @@"webpack-cli": "^4.10.0","webpack-dev-middleware": "^5.3.3","webpack-dev-server": "^4.11.1",+ "xml-js": "^1.6.11","yargs": "^17.6.2"},"engines": {@@ -5658,12 +5660,36 @@"node": ">= 6.0.0"}},+ "node_modules/chromedriver/node_modules/axios": {+ "version": "0.27.2",+ "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",+ "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",+ "dev": true,+ "dependencies": {+ "follow-redirects": "^1.14.9",+ "form-data": "^4.0.0"+ }+ },"node_modules/chromedriver/node_modules/compare-versions": {"version": "5.0.1","resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-5.0.1.tgz","integrity": "sha512-v8Au3l0b+Nwkp4G142JcgJFh1/TUhdxut7wzD1Nq1dyp5oa3tXaqb03EXOAB6jS4gMlalkjAUPZBMiAfKUixHQ==","dev": true},+ "node_modules/chromedriver/node_modules/form-data": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",+ "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",+ "dev": true,+ "dependencies": {+ "asynckit": "^0.4.0",+ "combined-stream": "^1.0.8",+ "mime-types": "^2.1.12"+ },+ "engines": {+ "node": ">= 6"+ }+ },"node_modules/chromedriver/node_modules/https-proxy-agent": {"version": "5.0.1","resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",@@ -19345,6 +19371,12 @@"node": ">=10"}},+ "node_modules/sax": {+ "version": "1.2.4",+ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",+ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",+ "dev": true+ },"node_modules/saxes": {"version": "5.0.1","resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",@@ -23173,6 +23205,18 @@}}},+ "node_modules/xml-js": {+ "version": "1.6.11",+ "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",+ "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==",+ "dev": true,+ "dependencies": {+ "sax": "^1.2.4"+ },+ "bin": {+ "xml-js": "bin/cli.js"+ }+ },"node_modules/xml-name-validator": {"version": "3.0.0","resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",@@ -27540,12 +27584,33 @@"debug": "4"}},+ "axios": {+ "version": "0.27.2",+ "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",+ "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",+ "dev": true,+ "requires": {+ "follow-redirects": "^1.14.9",+ "form-data": "^4.0.0"+ }+ },"compare-versions": {"version": "5.0.1","resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-5.0.1.tgz","integrity": "sha512-v8Au3l0b+Nwkp4G142JcgJFh1/TUhdxut7wzD1Nq1dyp5oa3tXaqb03EXOAB6jS4gMlalkjAUPZBMiAfKUixHQ==","dev": true},+ "form-data": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",+ "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",+ "dev": true,+ "requires": {+ "asynckit": "^0.4.0",+ "combined-stream": "^1.0.8",+ "mime-types": "^2.1.12"+ }+ },"https-proxy-agent": {"version": "5.0.1","resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",@@ -37974,6 +38039,12 @@}}},+ "sax": {+ "version": "1.2.4",+ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",+ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",+ "dev": true+ },"saxes": {"version": "5.0.1","resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",@@ -40954,6 +41025,15 @@"dev": true,"requires": {}},+ "xml-js": {+ "version": "1.6.11",+ "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",+ "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==",+ "dev": true,+ "requires": {+ "sax": "^1.2.4"+ }+ },"xml-name-validator": {"version": "3.0.0","resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",diff --git a/package.json b/package.jsonindex 85a30454f..3fc9e6beb 100644--- a/package.json+++ b/package.json@@ -96,6 +96,7 @@"@pmmmwh/react-refresh-webpack-plugin": "^0.5.7","@webpack-cli/serve": "^1.7.0","autoprefixer": "^10.4.12",+ "axios": "^1.2.1","babel-loader": "^8.1.0","browserstack-capabilities": "^0.7.0","browserstack-local": "^1.4.4",@@ -157,6 +158,7 @@"webpack-cli": "^4.10.0","webpack-dev-middleware": "^5.3.3","webpack-dev-server": "^4.11.1",+ "xml-js": "^1.6.11","yargs": "^17.6.2"},"dependencies": {diff --git a/tasks/bamboo/buildAll.sh b/tasks/bamboo/buildAll.shnew file mode 100755index 000000000..0b912c32a--- /dev/null+++ b/tasks/bamboo/buildAll.sh@@ -0,0 +1,20 @@+#!/bin/bash++# Bash commands to run in Bamboo to prepare an rpm for deployment++BASE=$(dirname "$0")/..++# Bamboo automatically sets NODE_ENV to production. Override this so that+# devDependencies are installed.+npm ci --production=false --ignore-scripts+node node_modules/.bin/patch-package --allow-root+# pipenv run bash $BASE/tasks/distExplict.sh++# Replacement of npm run dist+# Needed for bamboo to avoid permission errors with npm+node $BASE/tasks/clean.js+FETCH_GC=1 $BASE/tasks/buildOptions.sh+$BASE/tasks/buildOptions.sh+node $BASE/tasks/config.js+NODE_ENV=production npx webpack+node $BASE/tasks/dist.jsdiff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 696afdd97..a80ae1028 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -52,14 +52,14 @@ mkdir -p "$BUILD_DIR/colormaps"if [ "$FETCH_GC" ] ; thenrm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"- "$NODE_SCRIPTS_DIR/getCapabilities.js -c $OPT_DIR/$OPT_SUBDIR/config.json -gc $OPT_DIR/$OPT_SUBDIR/gc"+ `$NODE_SCRIPTS_DIR/getCapabilities.js --config "$OPT_DIR/$OPT_SUBDIR/config.json" --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`# Get visualization metadata (if configured)- rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- "$PYTHON_SCRIPTS_DIR/getVisMetadata.py" "$BUILD_DIR/features.json" \- "$BUILD_DIR/config/wv.json/layerOrder.json" "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"- exit 0+ # rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"+ # mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"+ # "$PYTHON_SCRIPTS_DIR/getVisMetadata.py" "$BUILD_DIR/features.json" \+ # "$BUILD_DIR/config/wv.json/layerOrder.json" "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"+ # exit 0fi# "$PYTHON_SCRIPTS_DIR/validateConfigs.py" "$SRC_DIR/common/config/wv.json/layers" \diff --git a/tasks/distExplict.sh b/tasks/distExplict.shdeleted file mode 100644index 036f85bec..000000000--- a/tasks/distExplict.sh+++ /dev/null@@ -1,12 +0,0 @@-#!/bin/bash--BASE=$(dirname "$0")/..-# Replacement of npm run dist-# Needed for bamboo to avoid-# permission errors with npm-node $BASE/tasks/clean.js-FETCH_GC=1 $BASE/tasks/buildOptions.sh-$BASE/tasks/buildOptions.sh-node $BASE/tasks/config.js-NODE_ENV=production npx webpack-node $BASE/tasks/dist.jsdiff --git a/tasks/node/getCapabilities.js b/tasks/node/getCapabilities.jsindex 690eaf42d..e868213dd 100755--- a/tasks/node/getCapabilities.js+++ b/tasks/node/getCapabilities.js@@ -1,43 +1,183 @@#!/usr/bin/env node+const fs = require('fs');+const path = require('path');const yargs = require('yargs');const console = require('console');+const request = require('axios').default;+const convert = require('xml-js');+const { promisify } = require('util');+const stream = require('stream');++const finished = promisify(stream.finished);+const prog = path.basename(__filename);+// const baseDir = path.join(__dirname, '..');const options = yargs.usage('Usage: $0 [options]').option('config', {+ demandOption: true,alias: 'c',type: 'string',description: 'config file',}).option('getcapabilities', {- alias: 'gc',+ demandOption: true,+ alias: 'g',type: 'string',description: 'getcapabilities file',- });+ })+ .epilog('Pulls GetCapabilities XML and linked metadata from configured locations');// import config file from ./config/default/release/config.json-async function main() {- console.log('testing');- const { argv } = options;+const { argv } = options;+if (!argv.config && !argv.getcapabilities) {+ console.error('Invalid number of arguments');+ process.exit(1);+}+const configFile = argv.config;+const outputDir = argv.getcapabilities;+const colormaps = {};+const vectorstyles = {};+const vectordata = {};+const colormapsDir = path.join(outputDir, 'colormaps');+const vectorstylesDir = path.join(outputDir, 'vectorstyles');+const vectordataDir = path.join(outputDir, 'vectordata');++const configData = fs.readFileSync(configFile);+const config = JSON.parse(configData);- if (!argv.config) {- console.log('error: config must be defined');- process.exit(1);+async function main () {+ if (!fs.existsSync(outputDir)) {+ fs.mkdirSync(outputDir);+ }++ await GetCapabilities();++ // process.exit();+}++async function GetCapabilities() {+ // Download each GC xml using the "from" attribute and put it in the "to" location+ if (Object.prototype.hasOwnProperty.call(config, 'wv-options-fetch')) {+ const fetchValues = config['wv-options-fetch'];+ Object.entries(fetchValues).forEach(async ([key, value]) => {+ const inputFile = value.from;+ const outputFile = `${outputDir}/${value.to}`;++ await fetchConfigs(inputFile, outputFile);+ await processGetCapabilities(outputFile);++ if (colormaps) {+ await gatherProcess(colormaps, 'colormaps', colormapsDir, '.xml');+ }+ if (vectorstyles) {+ await gatherProcess(vectorstyles, 'vectorstyles', vectorstylesDir, '.json');+ }+ if (vectordata) {+ await gatherProcess(vectordata, 'vectordatas', vectordataDir, '.json');+ }+ });}+}- argv.array.forEach((element) => {- console.log(element);+async function fetchConfigs(inputFile, outputFile) {+ const writer = await fs.createWriteStream(outputFile);+ return request({+ method: 'get',+ url: inputFile,+ responseType: 'stream',+ }).then(async (response) => {+ await response.data.pipe(writer);+ return finished(writer);});+}++async function handleException(error, link) {+ console.warn(`\n ${prog} WARN: Unable to fetch ${link} ${error}`);+}++async function processVectorData(layer) {+ if (layer['ows:Metadata']) {+ Object.values(layer['ows:Metadata']).forEach((item) => {+ const schemaVersion = item._attributes['xlink:role'];+ if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {+ const vectorDataLink = item._attributes['xlink:href'];+ const vectorDataFile = path.basename(vectorDataLink);+ const vectorDataId = path.parse(vectorDataFile).name;+ vectordata[vectorDataId] = vectorDataLink;+ }+ });+ }+}++async function processLayer(layer) {+ const ident = layer['ows:Identifier']._text;+ if (layer['ows:Metadata']) {+ if (config.skipPalettes) {+ console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`);+ } else {+ Object.values(layer['ows:Metadata']).forEach((item) => {+ const schemaVersion = item._attributes['xlink:role'];+ if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {+ const colormapLink = item._attributes['xlink:href'];+ const colormapFile = path.basename(colormapLink);+ const colormapId = path.parse(colormapFile).name;+ colormaps[colormapId] = colormapLink;+ } else if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0') {+ const vectorStyleLink = item._attributes['xlink:href'];+ const vectorStyleFile = path.basename(vectorStyleLink);+ const vectorStyleId = path.parse(vectorStyleFile).name;+ vectorstyles[vectorStyleId] = vectorStyleLink;+ }+ });+ }+ }+}++async function processGetCapabilities(outputFile) {+ const gcXML = fs.readFileSync(outputFile, { encoding: 'utf-8' });+ let gc = convert.xml2json(gcXML, { compact: true, spaces: 2 });+ gc = JSON.parse(gc);- // read config+ try {+ if (!gc.Capabilities || !gc.Capabilities.Contents) {+ console.error(`error: ${outputFile}: no layers`);+ process.exit(1);+ }- // get "wv-options-fetch" values+ const layers = gc.Capabilities.Contents.Layer;- // for each value+ Object.values(layers).forEach((layer) => {+ processLayer(layer);+ processVectorData(layer);+ });+ } catch (error) {+ console.error(`ERROR: ${outputFile}: ${error}`);+ process.exit(1);+ }+}++async function processMetadata(link, dir, ext) {+ try {+ const outputFile = path.join(dir, path.basename(link));+ if (link.endsWith(ext)) {+ await fetchConfigs(link, outputFile);+ }+ } catch (error) {+ handleException(error, link);+ }+}- // create an object of colormaps, vectorstyles, vectordata+async function gatherProcess(type, typeStr, dir, ext) {+ console.warn(`${prog}: Fetching ${Object.keys(type).length} ${typeStr}`);+ if (!fs.existsSync(dir)) {+ fs.mkdirSync(dir);+ }+ Object.values(type).forEach(async (link) => {+ await processMetadata(link, dir, ext);+ });}main().catch((err) => {