Interactive interface for browsing global, full-resolution satellite imagery
simplify granule fetching and caching (#5280)
Patrick Moulden91e632c8ec43
diff --git a/web/js/components/layer/settings/imagery-search.js b/web/js/components/layer/settings/imagery-search.jsindex 0de3c41b6..3668f4811 100644--- a/web/js/components/layer/settings/imagery-search.js+++ b/web/js/components/layer/settings/imagery-search.js@@ -12,6 +12,7 @@ const dateOptions = {};const parseGranuleTimestamp = (granule) => new Date(granule.time_start);const maxExtent = [-180, -90, 180, 90];+const headers = { 'Client-Id': 'Worldview' };export default function ImagerySearch({ layer }) {const listRef = useRef(null);@@ -46,7 +47,8 @@ export default function ImagerySearch({ layer }) {return maxExtent[i];});try {- const olderResponse = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=,${refDate.toISOString()}&sort_key=-start_date&pageSize=25&page_num=${pageNum}`);+ const olderUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=,${refDate.toISOString()}&sort_key=-start_date&pageSize=25&page_num=${pageNum}`;+ const olderResponse = await fetch(olderUrl, { headers });const olderGranules = await olderResponse.json();const olderDates = olderGranules.feed.entry.map(parseGranuleTimestamp);@@ -66,7 +68,8 @@ export default function ImagerySearch({ layer }) {return maxExtent[i];});try {- const newerResponse = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${refDate.toISOString()},&sort_key=start_date&pageSize=25&page_num=${pageNum}`);+ const newerUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${refDate.toISOString()},&sort_key=start_date&pageSize=25&page_num=${pageNum}`;+ const newerResponse = await fetch(newerUrl, { headers });const newerGranules = await newerResponse.json();const newerDates = newerGranules.feed.entry.map(parseGranuleTimestamp);@@ -99,8 +102,7 @@ export default function ImagerySearch({ layer }) {useEffect(() => {const asyncFunc = async () => {if (listRef.current.scrollHeight <= listRef.current.clientHeight) {- await loadOlderDates(layer, page);- await loadNewerDates(layer, page);+ await Promise.allSettled([loadOlderDates(layer, page), loadNewerDates(layer, page)]);setPage(page + 1);} else {listRef.current.scrollTop = listRef.current.scrollHeight / 2;diff --git a/web/js/containers/sidebar/layer-row.js b/web/js/containers/sidebar/layer-row.jsindex 68dd51f7a..7e9f99942 100644--- a/web/js/containers/sidebar/layer-row.js+++ b/web/js/containers/sidebar/layer-row.js@@ -148,11 +148,16 @@ function LayerRow (props) {}return maxExtent[i];});- const olderRes = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=P0Y0M0DT0H0M/${zeroedDate}&sort_key=-start_date&pageSize=1`);- const newerRes = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${zeroedDate}/P0Y0M1DT0H0M&sort_key=-start_date&pageSize=1`);+ const olderUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=P0Y0M0DT0H0M/${zeroedDate}&sort_key=-start_date&pageSize=1`;+ const newerUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${zeroedDate}/P0Y0M1DT0H0M&sort_key=-start_date&pageSize=1`;+ const headers = { 'Client-Id': 'Worldview' };+ const requests = [fetch(olderUrl, { headers }), fetch(newerUrl, { headers })];+ const responses = await Promise.allSettled(requests);+ const [olderRes, newerRes] = responses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);if (!olderRes.ok || !newerRes.ok) return;- const olderGranules = await olderRes.json();- const newerGranules = await newerRes.json();+ const jsonRequests = [olderRes.json(), newerRes.json()];+ const jsonResponses = await Promise.allSettled(jsonRequests);+ const [olderGranules, newerGranules] = jsonResponses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);const olderEntries = olderGranules?.feed?.entry || [];const newerEntries = newerGranules?.feed?.entry || [];const granules = [...olderEntries, ...newerEntries];diff --git a/web/js/map/granule/granule-layer-builder.js b/web/js/map/granule/granule-layer-builder.jsindex 8f1d1a97d..4e4b1d628 100644--- a/web/js/map/granule/granule-layer-builder.js+++ b/web/js/map/granule/granule-layer-builder.js@@ -1,5 +1,5 @@import OlLayerGroup from 'ol/layer/Group';-import { throttle as lodashThrottle, find } from 'lodash';+import { throttle as lodashThrottle } from 'lodash';import OlCollection from 'ol/Collection';import { DEFAULT_NUM_GRANULES } from '../../modules/layers/constants';import { updateGranuleLayerState } from '../../modules/layers/actions';@@ -26,16 +26,11 @@ import util from '../../util/util';const { toISOStringSeconds } = util;export default function granuleLayerBuilder(cache, store, createLayerWMTS) {- const CMRDataStore = {- [CRS.WEB_MERCATOR]: {},- [CRS.GEOGRAPHIC]: {},- [CRS.ANTARCTIC]: {},- [CRS.ARCTIC]: {},- };const getGranuleUrl = getGranulesUrlSelector(store.getState());const baseGranuleUrl = getGranuleUrl();const CMR_AJAX_OPTIONS = {url: baseGranuleUrl,+ cache: 'force-cache',headers: { 'Client-Id': 'Worldview' },traditional: true,dataType: 'json',@@ -64,62 +59,6 @@ export default function granuleLayerBuilder(cache, store, createLayerWMTS) {store.dispatch(stopLoading(LOADING_GRANULES));};- /**- * Collect granule metadata objects from a CMR response that have GIBS imagery- *- * @param {data} CMR data- * @param {id} layer id- */- const addGranuleCMRDateData = (def, data, shortName, dateRanges) => {- const { proj: { selected: { crs } } } = store.getState();- CMRDataStore[crs][shortName] = CMRDataStore[crs][shortName] || [];- const granuleData = CMRDataStore[crs][shortName];-- // Any granule metadata object that gets added here is assumed to have corresponding- // imagery in GIBS at the same time stamp. Therefore when determining whether or not- // to include a granule from CMR we see if either is true:- // - (imageryInDateRange) the date for the granule falls within a date range defined by GIBS for this layer- // - (imageryInTrailingRange) the date falls after the end date of the last date range AND before the layer's end date,- // or in the case of an ongoing layer with no endDate, now-- const lastDateRange = dateRanges[dateRanges.length - 1];- const finalEndDate = def.endDate || new Date();- data.forEach((entry) => {- const date = toISOStringSeconds(entry.time_start);- const imageryInDateRange = find(dateRanges, (r) => isWithinDateRange(date, r.startDate, r.endDate));- const imageryInTrailingRange = isWithinDateRange(date, lastDateRange.endDate, finalEndDate);- const existsForTime = find(granuleData, (g) => g.date === date);- if (!(imageryInDateRange || imageryInTrailingRange) || existsForTime) {- return;- }- const transformedGranule = transformGranuleData(entry, date, crs);- granuleData.push(transformedGranule);- });- granuleData.sort((a, b) => {- const dateA = new Date(a.date).valueOf();- const dateB = new Date(b.date).valueOf();- return dateB - dateA;- });- };-- const datesHaveBeenQueried = (start, end, existingGranules) => {- if (!existingGranules.length) return;- const latestDate = new Date(existingGranules[0].date);- const earliestDate = new Date(existingGranules[existingGranules.length - 1].date);- const startIsCovered = isWithinDateRange(start, earliestDate, latestDate);- const endIsCovered = isWithinDateRange(end, earliestDate, latestDate);- return startIsCovered && endIsCovered;- };-- const getGranulesFromStore = (shortName, nrtShortName) => {- const state = store.getState();- const { proj: { selected: { crs } } } = state;- const existingGranules = CMRDataStore[crs][shortName] || [];- const existingNRTGranules = CMRDataStore[crs][nrtShortName] || [];-- return [...existingGranules, ...existingNRTGranules];- };-/*** Query CMR to get dates* @param {object} def - Layer specs@@ -127,50 +66,41 @@ export default function granuleLayerBuilder(cache, store, createLayerWMTS) {*/const getQueriedGranuleDates = async (def, date) => {const {- endDate, startDate, title, visible, dateRanges,+ title,} = def;const state = store.getState();const { proj: { selected: { crs } } } = state;const getGranulesUrl = getGranulesUrlSelector(state);const params = getParamsForGranuleRequest(def, date, crs);const nrtParams = getParamsForGranuleRequest(def, date, crs, true);- const { shortName } = params;- const { shortName: nrtShortName } = nrtParams;let data = [];let nrtData = [];- const mergedExistingGranules = getGranulesFromStore(shortName, nrtShortName);- const datesQueried = datesHaveBeenQueried(params.startDate, date, mergedExistingGranules);-- if (mergedExistingGranules.length && datesQueried) {- return mergedExistingGranules;- }try {showLoading();const requestUrl = getGranulesUrl(params);const nrtRequestUrl = getGranulesUrl(nrtParams);- const response = await fetch(requestUrl, CMR_AJAX_OPTIONS);- const nrtResponse = await fetch(nrtRequestUrl, CMR_AJAX_OPTIONS);- data = await response.json();- nrtData = await nrtResponse.json();- data = data.feed.entry;- nrtData = nrtData.feed.entry;- if (data.length || nrtData.length) {- addGranuleCMRDateData(def, nrtData, nrtShortName, dateRanges);- addGranuleCMRDateData(def, data, shortName, dateRanges);- } else {- const dateWithinRange = isWithinDateRange(date, startDate, endDate);- // only show modal error if layer not set to hidden and outside of selected date range- if (visible && dateWithinRange) throttleDispathCMRErrorDialog(title);- }+ const requests = [fetch(requestUrl, CMR_AJAX_OPTIONS), fetch(nrtRequestUrl, CMR_AJAX_OPTIONS)];+ const responses = await Promise.allSettled(requests);+ const fulfilledResponses = responses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);+ const [response, nrtResponse] = fulfilledResponses;+ const jsonRequests = [response.json(), nrtResponse.json()];+ const jsonResponses = await Promise.allSettled(jsonRequests);+ const [responseJson, nrtResponseJson] = jsonResponses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);+ data = responseJson.feed.entry;+ nrtData = nrtResponseJson.feed.entry;} catch (e) {console.error(e);throttleDispathCMRErrorDialog(title);- return CMRDataStore[crs][shortName];} finally {hideLoading();}- return getGranulesFromStore(shortName, nrtShortName);+ const transformedData = [...data, ...nrtData].map((entry) => {+ const date = toISOStringSeconds(entry.time_start);++ return transformGranuleData(entry, date, crs);+ });+ return transformedData;};/**