Interactive interface for browsing global, full-resolution satellite imagery
fetchPreviewSnapshots arctic/antarctic wip
ZachTRice46f1f7ce3b36
diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.jsindex 31705903d..8bb333df8 100644--- a/tasks/build-options/fetchPreviewSnapshots.js+++ b/tasks/build-options/fetchPreviewSnapshots.js@@ -128,7 +128,12 @@ async function trackBadSnapshots (layerId, projection, request, imgFile, badSnapconst arcticBadSize = 9949const antarcticBadSize = 4060const geographicBadSize = 12088- const size = imgFile.tell()+ let size = 0++ fs.stat(imgFile, (err, stats) => {+ if (err) throw err+ size = stats.size+ })if (size === geographicBadSize || size === arcticBadSize || size === antarcticBadSize) {badSnapshots.push({@@ -143,10 +148,10 @@ async function getBestDate (projection, period, dateRanges) {const lastRange = dateRanges[dateRanges.length - 1]const startDate = lastRange.startDateconst endDate = lastRange.endDate- const parsedStartDate = moment(startDate, timeFormat)- const parsedEndDate = moment(endDate, timeFormat)- const pYear = parsedEndDate.year()- const pMonth = parsedEndDate.month()+ const parsedStartDate = moment(startDate)+ const parsedEndDate = moment(endDate)+ const parsedEndYear = parsedEndDate.year() + 1+ const parsedEndMonth = parsedEndDate.month() + 1let interval = parseInt(lastRange.dateInterval, 10)let alteredDate = null@@ -154,30 +159,30 @@ async function getBestDate (projection, period, dateRanges) {if (interval === 1) {interval = 3}- alteredDate = parsedEndDate.subtract(interval, 'days')+ alteredDate = moment(startDate).subtract(interval, 'days').utc().format()} else {- alteredDate = parsedEndDate.subtract(interval, 'months')+ alteredDate = moment(endDate).subtract(interval, 'months').utc().format()}// Choose a good daylight month for arctic- if (projection === 'arctic' && ![4, 5, 6, 7, 8, 9].includes(pMonth)) {+ if (projection === 'arctic' && ![4, 5, 6, 7, 8, 9].includes(parsedEndMonth)) {const currentYear = moment().year()const currentMonth = moment().month() + 1- if (pYear === currentYear && currentMonth < 6) {- alteredDate = moment({ year: pYear - 1, month: 5, day: 1 })+ if (parsedEndYear === currentYear && currentMonth < 6) {+ alteredDate = moment({ year: parsedEndYear - 1, month: 5, day: 1 }).utc().format()} else {- alteredDate = moment({ year: pYear, month: 5, day: 1 })+ alteredDate = moment({ year: parsedEndYear, month: 5, day: 1 }).utc().format()}}// Choose a good daylight month for antarctic- if (projection === 'antarctic' && ![10, 11, 12, 1, 2].includes(pMonth)) {- alteredDate = moment({ year: pYear, month: 11, day: 1 })+ if (projection === 'antarctic' && ![10, 11, 12, 1, 2].includes(parsedEndMonth)) {+ alteredDate = moment({ year: parsedEndYear, month: 11, day: 1 }).utc().format()}// Make sure modified date isn't out of layer date range- if (alteredDate && alteredDate.isSameOrAfter(parsedStartDate)) {- return alteredDate.format(timeFormat)+ if (alteredDate && moment(alteredDate).isSameOrAfter(parsedStartDate)) {+ return alteredDate}return endDate@@ -205,6 +210,7 @@ async function getSnapshots (layer) {for (const projection of Object.keys(layer.projections)) {const projDict = layer.projections[projection]const referenceLayer = referenceLayers[projection]+ const params = { ...paramDict.base, ...paramDict[projection] }// Sometimes a layer id is provided per projection (e.g. Land Mask layers)// We need to use this layer id to request the layer from WVS/GIBS@@ -217,7 +223,14 @@ async function getSnapshots (layer) {gibsLayerId = wvLayerId = layer.id}- const params = { ...paramDict.base, ...paramDict[projection] }+ const destFileName = path.join(destImgDir, projection, `${wvLayerId}.jpg`)++ // Only get images that we don't have already+ const fileExists = await fs.promises.stat(destFileName)+ .then(() => true)+ .catch(() => false)+ if (fileExists) continue+await getTimeParam(projection, wvLayerId, layer, params)if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {@@ -227,25 +240,28 @@ async function getSnapshots (layer) {params.LAYERS = gibsLayerId}- const destFileName = path.join(destImgDir, projection, `${wvLayerId}.jpg`)-- // Only get images that we don't have already- const fileExists = await fs.promises.stat(destFileName)- .then(() => true)- .catch(() => false)- if (fileExists) continue-try {- const imageReq = await axios.get(snapshotsUrl, { params })- let statusText = 'ERROR'+ const imageReq = await axios({+ method: 'get',+ url: snapshotsUrl,+ params,+ responseType: 'stream'+ })if (imageReq.status === 200) {statusText = 'SUCCESS'totalSuccessCount += 1- await fs.promises.writeFile(destFileName, imageReq.data, { flag: 'wx' })- if (gibsLayerId === referenceLayers[projection]) continue- await trackBadSnapshots(wvLayerId, projection, imageReq, imageReq.data)+ const dest = await fs.createWriteStream(destFileName, { flags: 'wx' })+ dest.on('finish', () => {+ console.warn(`File ${destFileName} has been written`)+ if (gibsLayerId === referenceLayers[projection]) {+ return+ }+ trackBadSnapshots(wvLayerId, projection, imageReq, destFileName)+ })+ imageReq.data.pipe(dest)} else {totalFailureCount += 1+ statusText = 'ERROR'}console.warn(`\n${prog}: Result: ${statusText} - ${imageReq.status}`)console.warn(`${prog}: Layer: ${wvLayerId}`)diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.shindex 8bc79470a..56c8b764b 100755--- a/tasks/buildOptions.sh+++ b/tasks/buildOptions.sh@@ -32,130 +32,130 @@ OPT_SUBDIR="${CONFIG_ENV-release}"CONFIG=$(echo $SRC_DIR | sed s_tasks/../__)echo "Using $CONFIG ($OPT_SUBDIR)"-# Copy options files to build directory, and various other file operations-rm -rf "$BUILD_DIR" "$DEST_DIR"-mkdir -p "$BUILD_DIR" "$DEST_DIR"--cp -r "$SRC_DIR"/common/* "$BUILD_DIR"--if hash rsync 2>/dev/null; then- rsync -a "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"-else- cp -r "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"-fi--mkdir -p "$DEST_DIR/config"-mkdir -p "$BUILD_DIR/colormaps"--# If $FETCH_GC is set, make various API requests-if [ "$FETCH_GC" ] ; then- # Fetch GC files and create colormaps, vectordata and vectorstyle files- rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"- rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"- `node $SCRIPTS_DIR/getCapabilities.js \- --config "$OPT_DIR/$OPT_SUBDIR/config.json" \- --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`-- # Get metadata for files in layerOrder.json and combine this data into 1 file- rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"- `node $SCRIPTS_DIR/getVisMetadata.js \- --features "$BUILD_DIR/features.json" \- --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \- --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`-else- # Validate layers in wv.json with a JSON schema- `node $SCRIPTS_DIR/validateConfigs.js \- --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \- --schemaFile "$BASE/schemas/layer-config.json"`-- if [ -e "$BUILD_DIR/features.json" ] ; then- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"- fi-- # Run extractConfigFromWMTS.js script with config.json- if [ -e "$BUILD_DIR/config.json" ] ; then- `node $SCRIPTS_DIR/extractConfigFromWMTS.js \- --config "$BUILD_DIR/config.json" \- --inputDir "$BUILD_DIR/gc" \- --outputDir "$BUILD_DIR/_wmts"`- fi-- # Run processVectorStyles.js and move vectorstyles where we want them- if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then- mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"- `node $SCRIPTS_DIR/processVectorStyles.js \- --inputDir "$BUILD_DIR/gc/vectorstyles" \- --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`- fi-- # Run processVectorData.js and move vectordata where we want them- if [ -e "$BUILD_DIR/gc/vectordata" ] ; then- mkdir -p "$BUILD_DIR/config/wv.json/vectordata"- `node $SCRIPTS_DIR/processVectorData.js \- --inputDir "$BUILD_DIR/gc/vectordata" \- --outputDir "$BUILD_DIR/config/wv.json/vectordata"`- fi-- # Run processColormap.js and move colormaps where we want them- if [ -e "$BUILD_DIR/colormaps" ] ; then- mkdir -p "$BUILD_DIR"/config/palettes- if [ -d "$BUILD_DIR"/gc/colormaps ] ; then- cp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc- fi- `node $SCRIPTS_DIR/processColormap.js \- --config "$OPT_DIR/$OPT_SUBDIR/config.json" \- --inputDir "$BUILD_DIR/colormaps" \- --outputDir "$BUILD_DIR/config/palettes"`- fi-- # Throw error if no categoryGroupOrder.json file present- if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then- echo "categoryGroupOrder.json not found. Generating..."- `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \- --inputDir "$SRC_DIR/common/config/wv.json/categories/" \- --outputDir "$SRC_DIR/common/config/wv.json/"`- fi-- if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then- cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"- fi-- # Run mergeConfig.js on each directory in /config containing .json files- # This creates on palettes-custom.json and wv.json- configs=$(ls "$BUILD_DIR/config")- for config in $configs; do- case $config in- *.json)- bash -c "node $SCRIPTS_DIR/mergeConfig.js \- --inputDir '$BUILD_DIR/config/$config' \- --outputFile '$DEST_DIR/config/$config'"- ;;- *)- cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"- ;;- esac- done-- # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json- `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \- --inputDir "$BUILD_DIR/_wmts" \- --outputFile "$DEST_DIR/config/wv.json"`-- # Copy brand files from build to dest- cp -r "$BUILD_DIR/brand" "$DEST_DIR"- cp "$BUILD_DIR/brand.json" "$DEST_DIR"--- # Validate the options build- `node $SCRIPTS_DIR/validateOptions.js \- --optionsFile "$BUILD_DIR/config.json" \- --configDir "$DEST_DIR/config"`+# # Copy options files to build directory, and various other file operations+# rm -rf "$BUILD_DIR" "$DEST_DIR"+# mkdir -p "$BUILD_DIR" "$DEST_DIR"++# cp -r "$SRC_DIR"/common/* "$BUILD_DIR"++# if hash rsync 2>/dev/null; then+# rsync -a "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"+# else+# cp -r "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"+# fi++# mkdir -p "$DEST_DIR/config"+# mkdir -p "$BUILD_DIR/colormaps"++# # If $FETCH_GC is set, make various API requests+# if [ "$FETCH_GC" ] ; then+# # Fetch GC files and create colormaps, vectordata and vectorstyle files+# rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"+# rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"+# `node $SCRIPTS_DIR/getCapabilities.js \+# --config "$OPT_DIR/$OPT_SUBDIR/config.json" \+# --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`++# # Get metadata for files in layerOrder.json and combine this data into 1 file+# rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"+# mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"+# `node $SCRIPTS_DIR/getVisMetadata.js \+# --features "$BUILD_DIR/features.json" \+# --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \+# --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`+# else+# # Validate layers in wv.json with a JSON schema+# `node $SCRIPTS_DIR/validateConfigs.js \+# --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \+# --schemaFile "$BASE/schemas/layer-config.json"`++# if [ -e "$BUILD_DIR/features.json" ] ; then+# cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"+# fi++# # Run extractConfigFromWMTS.js script with config.json+# if [ -e "$BUILD_DIR/config.json" ] ; then+# `node $SCRIPTS_DIR/extractConfigFromWMTS.js \+# --config "$BUILD_DIR/config.json" \+# --inputDir "$BUILD_DIR/gc" \+# --outputDir "$BUILD_DIR/_wmts"`+# fi++# # Run processVectorStyles.js and move vectorstyles where we want them+# if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then+# mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"+# `node $SCRIPTS_DIR/processVectorStyles.js \+# --inputDir "$BUILD_DIR/gc/vectorstyles" \+# --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`+# fi++# # Run processVectorData.js and move vectordata where we want them+# if [ -e "$BUILD_DIR/gc/vectordata" ] ; then+# mkdir -p "$BUILD_DIR/config/wv.json/vectordata"+# `node $SCRIPTS_DIR/processVectorData.js \+# --inputDir "$BUILD_DIR/gc/vectordata" \+# --outputDir "$BUILD_DIR/config/wv.json/vectordata"`+# fi++# # Run processColormap.js and move colormaps where we want them+# if [ -e "$BUILD_DIR/colormaps" ] ; then+# mkdir -p "$BUILD_DIR"/config/palettes+# if [ -d "$BUILD_DIR"/gc/colormaps ] ; then+# cp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc+# fi+# `node $SCRIPTS_DIR/processColormap.js \+# --config "$OPT_DIR/$OPT_SUBDIR/config.json" \+# --inputDir "$BUILD_DIR/colormaps" \+# --outputDir "$BUILD_DIR/config/palettes"`+# fi++# # Throw error if no categoryGroupOrder.json file present+# if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then+# echo "categoryGroupOrder.json not found. Generating..."+# `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \+# --inputDir "$SRC_DIR/common/config/wv.json/categories/" \+# --outputDir "$SRC_DIR/common/config/wv.json/"`+# fi++# if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then+# cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"+# fi++# # Run mergeConfig.js on each directory in /config containing .json files+# # This creates on palettes-custom.json and wv.json+# configs=$(ls "$BUILD_DIR/config")+# for config in $configs; do+# case $config in+# *.json)+# bash -c "node $SCRIPTS_DIR/mergeConfig.js \+# --inputDir '$BUILD_DIR/config/$config' \+# --outputFile '$DEST_DIR/config/$config'"+# ;;+# *)+# cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"+# ;;+# esac+# done++# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json+# `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \+# --inputDir "$BUILD_DIR/_wmts" \+# --outputFile "$DEST_DIR/config/wv.json"`++# # Copy brand files from build to dest+# cp -r "$BUILD_DIR/brand" "$DEST_DIR"+# cp "$BUILD_DIR/brand.json" "$DEST_DIR"+++# # Validate the options build+# `node $SCRIPTS_DIR/validateOptions.js \+# --optionsFile "$BUILD_DIR/config.json" \+# --configDir "$DEST_DIR/config"`# Fetch preview images from WV Snapshots for any layers which they are missing- # `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \- # --wvJsonFile "$DEST_DIR/config/wv.json" \- # --overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \- # --featuresFile "$BUILD_DIR/features.json"`-fi+ `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \+ --wvJsonFile "$DEST_DIR/config/wv.json" \+ --overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \+ --featuresFile "$BUILD_DIR/features.json"`+# fiexit 0