Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

fetchPreviewSnapshots arctic/antarctic wip

ZachTRice46f1f7ce3b36
diff --git a/tasks/build-options/fetchPreviewSnapshots.js b/tasks/build-options/fetchPreviewSnapshots.js
index 31705903d..8bb333df8 100644
--- a/tasks/build-options/fetchPreviewSnapshots.js
+++ b/tasks/build-options/fetchPreviewSnapshots.js
@@ -128,7 +128,12 @@ async function trackBadSnapshots (layerId, projection, request, imgFile, badSnap
const arcticBadSize = 9949
const antarcticBadSize = 4060
const geographicBadSize = 12088
- const size = imgFile.tell()
+ let size = 0
+
+ fs.stat(imgFile, (err, stats) => {
+ if (err) throw err
+ size = stats.size
+ })
if (size === geographicBadSize || size === arcticBadSize || size === antarcticBadSize) {
badSnapshots.push({
@@ -143,10 +148,10 @@ async function getBestDate (projection, period, dateRanges) {
const lastRange = dateRanges[dateRanges.length - 1]
const startDate = lastRange.startDate
const endDate = lastRange.endDate
- const parsedStartDate = moment(startDate, timeFormat)
- const parsedEndDate = moment(endDate, timeFormat)
- const pYear = parsedEndDate.year()
- const pMonth = parsedEndDate.month()
+ const parsedStartDate = moment(startDate)
+ const parsedEndDate = moment(endDate)
+ const parsedEndYear = parsedEndDate.year() + 1
+ const parsedEndMonth = parsedEndDate.month() + 1
let interval = parseInt(lastRange.dateInterval, 10)
let alteredDate = null
@@ -154,30 +159,30 @@ async function getBestDate (projection, period, dateRanges) {
if (interval === 1) {
interval = 3
}
- alteredDate = parsedEndDate.subtract(interval, 'days')
+ alteredDate = moment(startDate).subtract(interval, 'days').utc().format()
} else {
- alteredDate = parsedEndDate.subtract(interval, 'months')
+ alteredDate = moment(endDate).subtract(interval, 'months').utc().format()
}
// Choose a good daylight month for arctic
- if (projection === 'arctic' && ![4, 5, 6, 7, 8, 9].includes(pMonth)) {
+ if (projection === 'arctic' && ![4, 5, 6, 7, 8, 9].includes(parsedEndMonth)) {
const currentYear = moment().year()
const currentMonth = moment().month() + 1
- if (pYear === currentYear && currentMonth < 6) {
- alteredDate = moment({ year: pYear - 1, month: 5, day: 1 })
+ if (parsedEndYear === currentYear && currentMonth < 6) {
+ alteredDate = moment({ year: parsedEndYear - 1, month: 5, day: 1 }).utc().format()
} else {
- alteredDate = moment({ year: pYear, month: 5, day: 1 })
+ alteredDate = moment({ year: parsedEndYear, month: 5, day: 1 }).utc().format()
}
}
// Choose a good daylight month for antarctic
- if (projection === 'antarctic' && ![10, 11, 12, 1, 2].includes(pMonth)) {
- alteredDate = moment({ year: pYear, month: 11, day: 1 })
+ if (projection === 'antarctic' && ![10, 11, 12, 1, 2].includes(parsedEndMonth)) {
+ alteredDate = moment({ year: parsedEndYear, month: 11, day: 1 }).utc().format()
}
// Make sure modified date isn't out of layer date range
- if (alteredDate && alteredDate.isSameOrAfter(parsedStartDate)) {
- return alteredDate.format(timeFormat)
+ if (alteredDate && moment(alteredDate).isSameOrAfter(parsedStartDate)) {
+ return alteredDate
}
return endDate
@@ -205,6 +210,7 @@ async function getSnapshots (layer) {
for (const projection of Object.keys(layer.projections)) {
const projDict = layer.projections[projection]
const referenceLayer = referenceLayers[projection]
+ const params = { ...paramDict.base, ...paramDict[projection] }
// Sometimes a layer id is provided per projection (e.g. Land Mask layers)
// We need to use this layer id to request the layer from WVS/GIBS
@@ -217,7 +223,14 @@ async function getSnapshots (layer) {
gibsLayerId = wvLayerId = layer.id
}
- const params = { ...paramDict.base, ...paramDict[projection] }
+ const destFileName = path.join(destImgDir, projection, `${wvLayerId}.jpg`)
+
+ // Only get images that we don't have already
+ const fileExists = await fs.promises.stat(destFileName)
+ .then(() => true)
+ .catch(() => false)
+ if (fileExists) continue
+
await getTimeParam(projection, wvLayerId, layer, params)
if (gibsLayerId !== referenceLayer && !standaloneLayers.includes(gibsLayerId)) {
@@ -227,25 +240,28 @@ async function getSnapshots (layer) {
params.LAYERS = gibsLayerId
}
- const destFileName = path.join(destImgDir, projection, `${wvLayerId}.jpg`)
-
- // Only get images that we don't have already
- const fileExists = await fs.promises.stat(destFileName)
- .then(() => true)
- .catch(() => false)
- if (fileExists) continue
-
try {
- const imageReq = await axios.get(snapshotsUrl, { params })
- let statusText = 'ERROR'
+ const imageReq = await axios({
+ method: 'get',
+ url: snapshotsUrl,
+ params,
+ responseType: 'stream'
+ })
if (imageReq.status === 200) {
statusText = 'SUCCESS'
totalSuccessCount += 1
- await fs.promises.writeFile(destFileName, imageReq.data, { flag: 'wx' })
- if (gibsLayerId === referenceLayers[projection]) continue
- await trackBadSnapshots(wvLayerId, projection, imageReq, imageReq.data)
+ const dest = await fs.createWriteStream(destFileName, { flags: 'wx' })
+ dest.on('finish', () => {
+ console.warn(`File ${destFileName} has been written`)
+ if (gibsLayerId === referenceLayers[projection]) {
+ return
+ }
+ trackBadSnapshots(wvLayerId, projection, imageReq, destFileName)
+ })
+ imageReq.data.pipe(dest)
} else {
totalFailureCount += 1
+ statusText = 'ERROR'
}
console.warn(`\n${prog}: Result: ${statusText} - ${imageReq.status}`)
console.warn(`${prog}: Layer: ${wvLayerId}`)
diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
index 8bc79470a..56c8b764b 100755
--- a/tasks/buildOptions.sh
+++ b/tasks/buildOptions.sh
@@ -32,130 +32,130 @@ OPT_SUBDIR="${CONFIG_ENV-release}"
CONFIG=$(echo $SRC_DIR | sed s_tasks/../__)
echo "Using $CONFIG ($OPT_SUBDIR)"
-# Copy options files to build directory, and various other file operations
-rm -rf "$BUILD_DIR" "$DEST_DIR"
-mkdir -p "$BUILD_DIR" "$DEST_DIR"
-
-cp -r "$SRC_DIR"/common/* "$BUILD_DIR"
-
-if hash rsync 2>/dev/null; then
- rsync -a "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"
-else
- cp -r "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"
-fi
-
-mkdir -p "$DEST_DIR/config"
-mkdir -p "$BUILD_DIR/colormaps"
-
-# If $FETCH_GC is set, make various API requests
-if [ "$FETCH_GC" ] ; then
- # Fetch GC files and create colormaps, vectordata and vectorstyle files
- rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
- rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
- `node $SCRIPTS_DIR/getCapabilities.js \
- --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
- --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`
-
- # Get metadata for files in layerOrder.json and combine this data into 1 file
- rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
- mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
- `node $SCRIPTS_DIR/getVisMetadata.js \
- --features "$BUILD_DIR/features.json" \
- --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
- --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`
-else
- # Validate layers in wv.json with a JSON schema
- `node $SCRIPTS_DIR/validateConfigs.js \
- --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
- --schemaFile "$BASE/schemas/layer-config.json"`
-
- if [ -e "$BUILD_DIR/features.json" ] ; then
- cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
- fi
-
- # Run extractConfigFromWMTS.js script with config.json
- if [ -e "$BUILD_DIR/config.json" ] ; then
- `node $SCRIPTS_DIR/extractConfigFromWMTS.js \
- --config "$BUILD_DIR/config.json" \
- --inputDir "$BUILD_DIR/gc" \
- --outputDir "$BUILD_DIR/_wmts"`
- fi
-
- # Run processVectorStyles.js and move vectorstyles where we want them
- if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
- mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
- `node $SCRIPTS_DIR/processVectorStyles.js \
- --inputDir "$BUILD_DIR/gc/vectorstyles" \
- --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`
- fi
-
- # Run processVectorData.js and move vectordata where we want them
- if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
- mkdir -p "$BUILD_DIR/config/wv.json/vectordata"
- `node $SCRIPTS_DIR/processVectorData.js \
- --inputDir "$BUILD_DIR/gc/vectordata" \
- --outputDir "$BUILD_DIR/config/wv.json/vectordata"`
- fi
-
- # Run processColormap.js and move colormaps where we want them
- if [ -e "$BUILD_DIR/colormaps" ] ; then
- mkdir -p "$BUILD_DIR"/config/palettes
- if [ -d "$BUILD_DIR"/gc/colormaps ] ; then
- cp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc
- fi
- `node $SCRIPTS_DIR/processColormap.js \
- --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
- --inputDir "$BUILD_DIR/colormaps" \
- --outputDir "$BUILD_DIR/config/palettes"`
- fi
-
- # Throw error if no categoryGroupOrder.json file present
- if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then
- echo "categoryGroupOrder.json not found. Generating..."
- `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
- --inputDir "$SRC_DIR/common/config/wv.json/categories/" \
- --outputDir "$SRC_DIR/common/config/wv.json/"`
- fi
-
- if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then
- cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"
- fi
-
- # Run mergeConfig.js on each directory in /config containing .json files
- # This creates on palettes-custom.json and wv.json
- configs=$(ls "$BUILD_DIR/config")
- for config in $configs; do
- case $config in
- *.json)
- bash -c "node $SCRIPTS_DIR/mergeConfig.js \
- --inputDir '$BUILD_DIR/config/$config' \
- --outputFile '$DEST_DIR/config/$config'"
- ;;
- *)
- cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"
- ;;
- esac
- done
-
- # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
- `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
- --inputDir "$BUILD_DIR/_wmts" \
- --outputFile "$DEST_DIR/config/wv.json"`
-
- # Copy brand files from build to dest
- cp -r "$BUILD_DIR/brand" "$DEST_DIR"
- cp "$BUILD_DIR/brand.json" "$DEST_DIR"
-
-
- # Validate the options build
- `node $SCRIPTS_DIR/validateOptions.js \
- --optionsFile "$BUILD_DIR/config.json" \
- --configDir "$DEST_DIR/config"`
+# # Copy options files to build directory, and various other file operations
+# rm -rf "$BUILD_DIR" "$DEST_DIR"
+# mkdir -p "$BUILD_DIR" "$DEST_DIR"
+
+# cp -r "$SRC_DIR"/common/* "$BUILD_DIR"
+
+# if hash rsync 2>/dev/null; then
+# rsync -a "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"
+# else
+# cp -r "$SRC_DIR/$OPT_SUBDIR"/* "$BUILD_DIR"
+# fi
+
+# mkdir -p "$DEST_DIR/config"
+# mkdir -p "$BUILD_DIR/colormaps"
+
+# # If $FETCH_GC is set, make various API requests
+# if [ "$FETCH_GC" ] ; then
+# # Fetch GC files and create colormaps, vectordata and vectorstyle files
+# rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
+# rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
+# `node $SCRIPTS_DIR/getCapabilities.js \
+# --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
+# --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`
+
+# # Get metadata for files in layerOrder.json and combine this data into 1 file
+# rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
+# mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
+# `node $SCRIPTS_DIR/getVisMetadata.js \
+# --features "$BUILD_DIR/features.json" \
+# --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
+# --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`
+# else
+# # Validate layers in wv.json with a JSON schema
+# `node $SCRIPTS_DIR/validateConfigs.js \
+# --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
+# --schemaFile "$BASE/schemas/layer-config.json"`
+
+# if [ -e "$BUILD_DIR/features.json" ] ; then
+# cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
+# fi
+
+# # Run extractConfigFromWMTS.js script with config.json
+# if [ -e "$BUILD_DIR/config.json" ] ; then
+# `node $SCRIPTS_DIR/extractConfigFromWMTS.js \
+# --config "$BUILD_DIR/config.json" \
+# --inputDir "$BUILD_DIR/gc" \
+# --outputDir "$BUILD_DIR/_wmts"`
+# fi
+
+# # Run processVectorStyles.js and move vectorstyles where we want them
+# if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
+# mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
+# `node $SCRIPTS_DIR/processVectorStyles.js \
+# --inputDir "$BUILD_DIR/gc/vectorstyles" \
+# --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`
+# fi
+
+# # Run processVectorData.js and move vectordata where we want them
+# if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
+# mkdir -p "$BUILD_DIR/config/wv.json/vectordata"
+# `node $SCRIPTS_DIR/processVectorData.js \
+# --inputDir "$BUILD_DIR/gc/vectordata" \
+# --outputDir "$BUILD_DIR/config/wv.json/vectordata"`
+# fi
+
+# # Run processColormap.js and move colormaps where we want them
+# if [ -e "$BUILD_DIR/colormaps" ] ; then
+# mkdir -p "$BUILD_DIR"/config/palettes
+# if [ -d "$BUILD_DIR"/gc/colormaps ] ; then
+# cp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc
+# fi
+# `node $SCRIPTS_DIR/processColormap.js \
+# --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
+# --inputDir "$BUILD_DIR/colormaps" \
+# --outputDir "$BUILD_DIR/config/palettes"`
+# fi
+
+# # Throw error if no categoryGroupOrder.json file present
+# if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then
+# echo "categoryGroupOrder.json not found. Generating..."
+# `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
+# --inputDir "$SRC_DIR/common/config/wv.json/categories/" \
+# --outputDir "$SRC_DIR/common/config/wv.json/"`
+# fi
+
+# if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then
+# cp "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" "$BUILD_DIR/config/wv.json/layer-metadata.json"
+# fi
+
+# # Run mergeConfig.js on each directory in /config containing .json files
+# # This creates on palettes-custom.json and wv.json
+# configs=$(ls "$BUILD_DIR/config")
+# for config in $configs; do
+# case $config in
+# *.json)
+# bash -c "node $SCRIPTS_DIR/mergeConfig.js \
+# --inputDir '$BUILD_DIR/config/$config' \
+# --outputFile '$DEST_DIR/config/$config'"
+# ;;
+# *)
+# cp -r "$BUILD_DIR/config/$config" "$DEST_DIR/config/$config"
+# ;;
+# esac
+# done
+
+# # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
+# `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
+# --inputDir "$BUILD_DIR/_wmts" \
+# --outputFile "$DEST_DIR/config/wv.json"`
+
+# # Copy brand files from build to dest
+# cp -r "$BUILD_DIR/brand" "$DEST_DIR"
+# cp "$BUILD_DIR/brand.json" "$DEST_DIR"
+
+
+# # Validate the options build
+# `node $SCRIPTS_DIR/validateOptions.js \
+# --optionsFile "$BUILD_DIR/config.json" \
+# --configDir "$DEST_DIR/config"`
# Fetch preview images from WV Snapshots for any layers which they are missing
- # `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
- # --wvJsonFile "$DEST_DIR/config/wv.json" \
- # --overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \
- # --featuresFile "$BUILD_DIR/features.json"`
-fi
+ `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
+ --wvJsonFile "$DEST_DIR/config/wv.json" \
+ --overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \
+ --featuresFile "$BUILD_DIR/features.json"`
+# fi
exit 0