Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

convert getCapabilities + cleanup bamboo & ci/cd scripts

ZachTRice9d05319f24dd
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index 2d48f5748..b9a94baf3 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -12,19 +12,14 @@ jobs:
fail-fast: false
matrix:
os: [macOS-latest, ubuntu-latest, windows-latest]
- node: [18]
+ node: [16]
steps:
- name: Checkout the code
- uses: actions/checkout@v2
-
- - name: Set up Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.9
+ uses: actions/checkout@v3
- name: Setup Node.js environment
- uses: actions/[email protected]
+ uses: actions/setup-node@v3
with:
node-version: ${{matrix.node}}
diff --git a/.github/workflows/layer-pr-template.yml b/.github/workflows/layer-pr-template.yml
index 5025da632..2c3c71e84 100644
--- a/.github/workflows/layer-pr-template.yml
+++ b/.github/workflows/layer-pr-template.yml
@@ -11,7 +11,7 @@ jobs:
name: 'Create PR with Layers Template'
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: pull-request
uses: repo-sync/pull-request@v2
with:
diff --git a/legacy/tasks/buildAll.sh b/legacy/tasks/buildAll.sh
deleted file mode 100755
index 965becf60..000000000
--- a/legacy/tasks/buildAll.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-BASE=$(dirname "$0")/..
-
-# Bash commands to run in Bamboo to prepare an rpm for deployment
-
-# Overwrite the placeholder config in the RPM directory with the actual
-# bit.ly configuration
-cp bitly.json rpm/bitly.json
-# Bamboo automatically sets NODE_ENV to production. Override this so that
-# devDependencies are installed.
-
-npm ci --production=false --ignore-scripts
-node node_modules/.bin/patch-package --allow-root
-$BASE/tasks/pythonInstall.sh linux
-pipenv run bash $BASE/tasks/distExplict.sh
-node $BASE/rpm/buildRPM.js
-
-# This is where bamboo expects to find artifacts
-mv ~/rpmbuild build/rpm
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 20c87478f..cb8003c3a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -82,6 +82,7 @@
"@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
"@webpack-cli/serve": "^1.7.0",
"autoprefixer": "^10.4.12",
+ "axios": "^1.2.1",
"babel-loader": "^8.1.0",
"browserstack-capabilities": "^0.7.0",
"browserstack-local": "^1.4.4",
@@ -143,6 +144,7 @@
"webpack-cli": "^4.10.0",
"webpack-dev-middleware": "^5.3.3",
"webpack-dev-server": "^4.11.1",
+ "xml-js": "^1.6.11",
"yargs": "^17.6.2"
},
"engines": {
@@ -5658,12 +5660,36 @@
"node": ">= 6.0.0"
}
},
+ "node_modules/chromedriver/node_modules/axios": {
+ "version": "0.27.2",
+ "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",
+ "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
+ "dev": true,
+ "dependencies": {
+ "follow-redirects": "^1.14.9",
+ "form-data": "^4.0.0"
+ }
+ },
"node_modules/chromedriver/node_modules/compare-versions": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-5.0.1.tgz",
"integrity": "sha512-v8Au3l0b+Nwkp4G142JcgJFh1/TUhdxut7wzD1Nq1dyp5oa3tXaqb03EXOAB6jS4gMlalkjAUPZBMiAfKUixHQ==",
"dev": true
},
+ "node_modules/chromedriver/node_modules/form-data": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+ "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+ "dev": true,
+ "dependencies": {
+ "asynckit": "^0.4.0",
+ "combined-stream": "^1.0.8",
+ "mime-types": "^2.1.12"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/chromedriver/node_modules/https-proxy-agent": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
@@ -19345,6 +19371,12 @@
"node": ">=10"
}
},
+ "node_modules/sax": {
+ "version": "1.2.4",
+ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",
+ "dev": true
+ },
"node_modules/saxes": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",
@@ -23173,6 +23205,18 @@
}
}
},
+ "node_modules/xml-js": {
+ "version": "1.6.11",
+ "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",
+ "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==",
+ "dev": true,
+ "dependencies": {
+ "sax": "^1.2.4"
+ },
+ "bin": {
+ "xml-js": "bin/cli.js"
+ }
+ },
"node_modules/xml-name-validator": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
@@ -27540,12 +27584,33 @@
"debug": "4"
}
},
+ "axios": {
+ "version": "0.27.2",
+ "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",
+ "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
+ "dev": true,
+ "requires": {
+ "follow-redirects": "^1.14.9",
+ "form-data": "^4.0.0"
+ }
+ },
"compare-versions": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-5.0.1.tgz",
"integrity": "sha512-v8Au3l0b+Nwkp4G142JcgJFh1/TUhdxut7wzD1Nq1dyp5oa3tXaqb03EXOAB6jS4gMlalkjAUPZBMiAfKUixHQ==",
"dev": true
},
+ "form-data": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+ "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+ "dev": true,
+ "requires": {
+ "asynckit": "^0.4.0",
+ "combined-stream": "^1.0.8",
+ "mime-types": "^2.1.12"
+ }
+ },
"https-proxy-agent": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
@@ -37974,6 +38039,12 @@
}
}
},
+ "sax": {
+ "version": "1.2.4",
+ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",
+ "dev": true
+ },
"saxes": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",
@@ -40954,6 +41025,15 @@
"dev": true,
"requires": {}
},
+ "xml-js": {
+ "version": "1.6.11",
+ "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",
+ "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==",
+ "dev": true,
+ "requires": {
+ "sax": "^1.2.4"
+ }
+ },
"xml-name-validator": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
diff --git a/package.json b/package.json
index 85a30454f..3fc9e6beb 100644
--- a/package.json
+++ b/package.json
@@ -96,6 +96,7 @@
"@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
"@webpack-cli/serve": "^1.7.0",
"autoprefixer": "^10.4.12",
+ "axios": "^1.2.1",
"babel-loader": "^8.1.0",
"browserstack-capabilities": "^0.7.0",
"browserstack-local": "^1.4.4",
@@ -157,6 +158,7 @@
"webpack-cli": "^4.10.0",
"webpack-dev-middleware": "^5.3.3",
"webpack-dev-server": "^4.11.1",
+ "xml-js": "^1.6.11",
"yargs": "^17.6.2"
},
"dependencies": {
diff --git a/tasks/bamboo/buildAll.sh b/tasks/bamboo/buildAll.sh
new file mode 100755
index 000000000..0b912c32a
--- /dev/null
+++ b/tasks/bamboo/buildAll.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Bash commands to run in Bamboo to prepare an rpm for deployment
+
+BASE=$(dirname "$0")/..
+
+# Bamboo automatically sets NODE_ENV to production. Override this so that
+# devDependencies are installed.
+npm ci --production=false --ignore-scripts
+node node_modules/.bin/patch-package --allow-root
+# pipenv run bash $BASE/tasks/distExplict.sh
+
+# Replacement of npm run dist
+# Needed for bamboo to avoid permission errors with npm
+node $BASE/tasks/clean.js
+FETCH_GC=1 $BASE/tasks/buildOptions.sh
+$BASE/tasks/buildOptions.sh
+node $BASE/tasks/config.js
+NODE_ENV=production npx webpack
+node $BASE/tasks/dist.js
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 696afdd97..a80ae1028 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -52,14 +52,14 @@ mkdir -p "$BUILD_DIR/colormaps"
if [ "$FETCH_GC" ] ; then
rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
- "$NODE_SCRIPTS_DIR/getCapabilities.js -c $OPT_DIR/$OPT_SUBDIR/config.json -gc $OPT_DIR/$OPT_SUBDIR/gc"
+ `$NODE_SCRIPTS_DIR/getCapabilities.js --config "$OPT_DIR/$OPT_SUBDIR/config.json" --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`
# Get visualization metadata (if configured)
- rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
- mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
- "$PYTHON_SCRIPTS_DIR/getVisMetadata.py" "$BUILD_DIR/features.json" \
- "$BUILD_DIR/config/wv.json/layerOrder.json" "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"
- exit 0
+ # rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
+ # mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
+ # "$PYTHON_SCRIPTS_DIR/getVisMetadata.py" "$BUILD_DIR/features.json" \
+ # "$BUILD_DIR/config/wv.json/layerOrder.json" "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"
+ # exit 0
fi
# "$PYTHON_SCRIPTS_DIR/validateConfigs.py" "$SRC_DIR/common/config/wv.json/layers" \
diff --git a/tasks/distExplict.sh b/tasks/distExplict.sh
deleted file mode 100644
index 036f85bec..000000000
--- a/tasks/distExplict.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-BASE=$(dirname "$0")/..
-# Replacement of npm run dist
-# Needed for bamboo to avoid
-# permission errors with npm
-node $BASE/tasks/clean.js
-FETCH_GC=1 $BASE/tasks/buildOptions.sh
-$BASE/tasks/buildOptions.sh
-node $BASE/tasks/config.js
-NODE_ENV=production npx webpack
-node $BASE/tasks/dist.js
diff --git a/tasks/node/getCapabilities.js b/tasks/node/getCapabilities.js
index 690eaf42d..e868213dd 100755
--- a/tasks/node/getCapabilities.js
+++ b/tasks/node/getCapabilities.js
@@ -1,43 +1,183 @@
#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
const yargs = require('yargs');
const console = require('console');
+const request = require('axios').default;
+const convert = require('xml-js');
+const { promisify } = require('util');
+const stream = require('stream');
+
+const finished = promisify(stream.finished);
+const prog = path.basename(__filename);
+// const baseDir = path.join(__dirname, '..');
const options = yargs
.usage('Usage: $0 [options]')
.option('config', {
+ demandOption: true,
alias: 'c',
type: 'string',
description: 'config file',
})
.option('getcapabilities', {
- alias: 'gc',
+ demandOption: true,
+ alias: 'g',
type: 'string',
description: 'getcapabilities file',
- });
+ })
+ .epilog('Pulls GetCapabilities XML and linked metadata from configured locations');
// import config file from ./config/default/release/config.json
-async function main() {
- console.log('testing');
- const { argv } = options;
+const { argv } = options;
+if (!argv.config && !argv.getcapabilities) {
+ console.error('Invalid number of arguments');
+ process.exit(1);
+}
+const configFile = argv.config;
+const outputDir = argv.getcapabilities;
+const colormaps = {};
+const vectorstyles = {};
+const vectordata = {};
+const colormapsDir = path.join(outputDir, 'colormaps');
+const vectorstylesDir = path.join(outputDir, 'vectorstyles');
+const vectordataDir = path.join(outputDir, 'vectordata');
+
+const configData = fs.readFileSync(configFile);
+const config = JSON.parse(configData);
- if (!argv.config) {
- console.log('error: config must be defined');
- process.exit(1);
+async function main () {
+ if (!fs.existsSync(outputDir)) {
+ fs.mkdirSync(outputDir);
+ }
+
+ await GetCapabilities();
+
+ // process.exit();
+}
+
+async function GetCapabilities() {
+ // Download each GC xml using the "from" attribute and put it in the "to" location
+ if (Object.prototype.hasOwnProperty.call(config, 'wv-options-fetch')) {
+ const fetchValues = config['wv-options-fetch'];
+ Object.entries(fetchValues).forEach(async ([key, value]) => {
+ const inputFile = value.from;
+ const outputFile = `${outputDir}/${value.to}`;
+
+ await fetchConfigs(inputFile, outputFile);
+ await processGetCapabilities(outputFile);
+
+ if (colormaps) {
+ await gatherProcess(colormaps, 'colormaps', colormapsDir, '.xml');
+ }
+ if (vectorstyles) {
+ await gatherProcess(vectorstyles, 'vectorstyles', vectorstylesDir, '.json');
+ }
+ if (vectordata) {
+ await gatherProcess(vectordata, 'vectordatas', vectordataDir, '.json');
+ }
+ });
}
+}
- argv.array.forEach((element) => {
- console.log(element);
+async function fetchConfigs(inputFile, outputFile) {
+ const writer = await fs.createWriteStream(outputFile);
+ return request({
+ method: 'get',
+ url: inputFile,
+ responseType: 'stream',
+ }).then(async (response) => {
+ await response.data.pipe(writer);
+ return finished(writer);
});
+}
+
+async function handleException(error, link) {
+ console.warn(`\n ${prog} WARN: Unable to fetch ${link} ${error}`);
+}
+
+async function processVectorData(layer) {
+ if (layer['ows:Metadata']) {
+ Object.values(layer['ows:Metadata']).forEach((item) => {
+ const schemaVersion = item._attributes['xlink:role'];
+ if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {
+ const vectorDataLink = item._attributes['xlink:href'];
+ const vectorDataFile = path.basename(vectorDataLink);
+ const vectorDataId = path.parse(vectorDataFile).name;
+ vectordata[vectorDataId] = vectorDataLink;
+ }
+ });
+ }
+}
+
+async function processLayer(layer) {
+ const ident = layer['ows:Identifier']._text;
+ if (layer['ows:Metadata']) {
+ if (config.skipPalettes) {
+ console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`);
+ } else {
+ Object.values(layer['ows:Metadata']).forEach((item) => {
+ const schemaVersion = item._attributes['xlink:role'];
+ if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {
+ const colormapLink = item._attributes['xlink:href'];
+ const colormapFile = path.basename(colormapLink);
+ const colormapId = path.parse(colormapFile).name;
+ colormaps[colormapId] = colormapLink;
+ } else if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/mapbox-gl-style/1.0') {
+ const vectorStyleLink = item._attributes['xlink:href'];
+ const vectorStyleFile = path.basename(vectorStyleLink);
+ const vectorStyleId = path.parse(vectorStyleFile).name;
+ vectorstyles[vectorStyleId] = vectorStyleLink;
+ }
+ });
+ }
+ }
+}
+
+async function processGetCapabilities(outputFile) {
+ const gcXML = fs.readFileSync(outputFile, { encoding: 'utf-8' });
+ let gc = convert.xml2json(gcXML, { compact: true, spaces: 2 });
+ gc = JSON.parse(gc);
- // read config
+ try {
+ if (!gc.Capabilities || !gc.Capabilities.Contents) {
+ console.error(`error: ${outputFile}: no layers`);
+ process.exit(1);
+ }
- // get "wv-options-fetch" values
+ const layers = gc.Capabilities.Contents.Layer;
- // for each value
+ Object.values(layers).forEach((layer) => {
+ processLayer(layer);
+ processVectorData(layer);
+ });
+ } catch (error) {
+ console.error(`ERROR: ${outputFile}: ${error}`);
+ process.exit(1);
+ }
+}
+
+async function processMetadata(link, dir, ext) {
+ try {
+ const outputFile = path.join(dir, path.basename(link));
+ if (link.endsWith(ext)) {
+ await fetchConfigs(link, outputFile);
+ }
+ } catch (error) {
+ handleException(error, link);
+ }
+}
- // create an object of colormaps, vectorstyles, vectordata
+async function gatherProcess(type, typeStr, dir, ext) {
+ console.warn(`${prog}: Fetching ${Object.keys(type).length} ${typeStr}`);
+ if (!fs.existsSync(dir)) {
+ fs.mkdirSync(dir);
+ }
+ Object.values(type).forEach(async (link) => {
+ await processMetadata(link, dir, ext);
+ });
}
main().catch((err) => {