Skip to content

Interactive interface for browsing global, full-resolution satellite imagery

simplify granule fetching and caching (#5280)

Patrick Moulden91e632c8ec43
diff --git a/web/js/components/layer/settings/imagery-search.js b/web/js/components/layer/settings/imagery-search.js
index 0de3c41b6..3668f4811 100644
--- a/web/js/components/layer/settings/imagery-search.js
+++ b/web/js/components/layer/settings/imagery-search.js
@@ -12,6 +12,7 @@ const dateOptions = {
};
const parseGranuleTimestamp = (granule) => new Date(granule.time_start);
const maxExtent = [-180, -90, 180, 90];
+const headers = { 'Client-Id': 'Worldview' };
export default function ImagerySearch({ layer }) {
const listRef = useRef(null);
@@ -46,7 +47,8 @@ export default function ImagerySearch({ layer }) {
return maxExtent[i];
});
try {
- const olderResponse = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=,${refDate.toISOString()}&sort_key=-start_date&pageSize=25&page_num=${pageNum}`);
+ const olderUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=,${refDate.toISOString()}&sort_key=-start_date&pageSize=25&page_num=${pageNum}`;
+ const olderResponse = await fetch(olderUrl, { headers });
const olderGranules = await olderResponse.json();
const olderDates = olderGranules.feed.entry.map(parseGranuleTimestamp);
@@ -66,7 +68,8 @@ export default function ImagerySearch({ layer }) {
return maxExtent[i];
});
try {
- const newerResponse = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${refDate.toISOString()},&sort_key=start_date&pageSize=25&page_num=${pageNum}`);
+ const newerUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${refDate.toISOString()},&sort_key=start_date&pageSize=25&page_num=${pageNum}`;
+ const newerResponse = await fetch(newerUrl, { headers });
const newerGranules = await newerResponse.json();
const newerDates = newerGranules.feed.entry.map(parseGranuleTimestamp);
@@ -99,8 +102,7 @@ export default function ImagerySearch({ layer }) {
useEffect(() => {
const asyncFunc = async () => {
if (listRef.current.scrollHeight <= listRef.current.clientHeight) {
- await loadOlderDates(layer, page);
- await loadNewerDates(layer, page);
+ await Promise.allSettled([loadOlderDates(layer, page), loadNewerDates(layer, page)]);
setPage(page + 1);
} else {
listRef.current.scrollTop = listRef.current.scrollHeight / 2;
diff --git a/web/js/containers/sidebar/layer-row.js b/web/js/containers/sidebar/layer-row.js
index 68dd51f7a..7e9f99942 100644
--- a/web/js/containers/sidebar/layer-row.js
+++ b/web/js/containers/sidebar/layer-row.js
@@ -148,11 +148,16 @@ function LayerRow (props) {
}
return maxExtent[i];
});
- const olderRes = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=P0Y0M0DT0H0M/${zeroedDate}&sort_key=-start_date&pageSize=1`);
- const newerRes = await fetch(`https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${zeroedDate}/P0Y0M1DT0H0M&sort_key=-start_date&pageSize=1`);
+ const olderUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=P0Y0M0DT0H0M/${zeroedDate}&sort_key=-start_date&pageSize=1`;
+ const newerUrl = `https://cmr.earthdata.nasa.gov/search/granules.json?collection_concept_id=${conceptID}&bounding_box=${extent.join(',')}&temporal=${zeroedDate}/P0Y0M1DT0H0M&sort_key=-start_date&pageSize=1`;
+ const headers = { 'Client-Id': 'Worldview' };
+ const requests = [fetch(olderUrl, { headers }), fetch(newerUrl, { headers })];
+ const responses = await Promise.allSettled(requests);
+ const [olderRes, newerRes] = responses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);
if (!olderRes.ok || !newerRes.ok) return;
- const olderGranules = await olderRes.json();
- const newerGranules = await newerRes.json();
+ const jsonRequests = [olderRes.json(), newerRes.json()];
+ const jsonResponses = await Promise.allSettled(jsonRequests);
+ const [olderGranules, newerGranules] = jsonResponses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);
const olderEntries = olderGranules?.feed?.entry || [];
const newerEntries = newerGranules?.feed?.entry || [];
const granules = [...olderEntries, ...newerEntries];
diff --git a/web/js/map/granule/granule-layer-builder.js b/web/js/map/granule/granule-layer-builder.js
index 8f1d1a97d..4e4b1d628 100644
--- a/web/js/map/granule/granule-layer-builder.js
+++ b/web/js/map/granule/granule-layer-builder.js
@@ -1,5 +1,5 @@
import OlLayerGroup from 'ol/layer/Group';
-import { throttle as lodashThrottle, find } from 'lodash';
+import { throttle as lodashThrottle } from 'lodash';
import OlCollection from 'ol/Collection';
import { DEFAULT_NUM_GRANULES } from '../../modules/layers/constants';
import { updateGranuleLayerState } from '../../modules/layers/actions';
@@ -26,16 +26,11 @@ import util from '../../util/util';
const { toISOStringSeconds } = util;
export default function granuleLayerBuilder(cache, store, createLayerWMTS) {
- const CMRDataStore = {
- [CRS.WEB_MERCATOR]: {},
- [CRS.GEOGRAPHIC]: {},
- [CRS.ANTARCTIC]: {},
- [CRS.ARCTIC]: {},
- };
const getGranuleUrl = getGranulesUrlSelector(store.getState());
const baseGranuleUrl = getGranuleUrl();
const CMR_AJAX_OPTIONS = {
url: baseGranuleUrl,
+ cache: 'force-cache',
headers: { 'Client-Id': 'Worldview' },
traditional: true,
dataType: 'json',
@@ -64,62 +59,6 @@ export default function granuleLayerBuilder(cache, store, createLayerWMTS) {
store.dispatch(stopLoading(LOADING_GRANULES));
};
- /**
- * Collect granule metadata objects from a CMR response that have GIBS imagery
- *
- * @param {data} CMR data
- * @param {id} layer id
- */
- const addGranuleCMRDateData = (def, data, shortName, dateRanges) => {
- const { proj: { selected: { crs } } } = store.getState();
- CMRDataStore[crs][shortName] = CMRDataStore[crs][shortName] || [];
- const granuleData = CMRDataStore[crs][shortName];
-
- // Any granule metadata object that gets added here is assumed to have corresponding
- // imagery in GIBS at the same time stamp. Therefore when determining whether or not
- // to include a granule from CMR we see if either is true:
- // - (imageryInDateRange) the date for the granule falls within a date range defined by GIBS for this layer
- // - (imageryInTrailingRange) the date falls after the end date of the last date range AND before the layer's end date,
- // or in the case of an ongoing layer with no endDate, now
-
- const lastDateRange = dateRanges[dateRanges.length - 1];
- const finalEndDate = def.endDate || new Date();
- data.forEach((entry) => {
- const date = toISOStringSeconds(entry.time_start);
- const imageryInDateRange = find(dateRanges, (r) => isWithinDateRange(date, r.startDate, r.endDate));
- const imageryInTrailingRange = isWithinDateRange(date, lastDateRange.endDate, finalEndDate);
- const existsForTime = find(granuleData, (g) => g.date === date);
- if (!(imageryInDateRange || imageryInTrailingRange) || existsForTime) {
- return;
- }
- const transformedGranule = transformGranuleData(entry, date, crs);
- granuleData.push(transformedGranule);
- });
- granuleData.sort((a, b) => {
- const dateA = new Date(a.date).valueOf();
- const dateB = new Date(b.date).valueOf();
- return dateB - dateA;
- });
- };
-
- const datesHaveBeenQueried = (start, end, existingGranules) => {
- if (!existingGranules.length) return;
- const latestDate = new Date(existingGranules[0].date);
- const earliestDate = new Date(existingGranules[existingGranules.length - 1].date);
- const startIsCovered = isWithinDateRange(start, earliestDate, latestDate);
- const endIsCovered = isWithinDateRange(end, earliestDate, latestDate);
- return startIsCovered && endIsCovered;
- };
-
- const getGranulesFromStore = (shortName, nrtShortName) => {
- const state = store.getState();
- const { proj: { selected: { crs } } } = state;
- const existingGranules = CMRDataStore[crs][shortName] || [];
- const existingNRTGranules = CMRDataStore[crs][nrtShortName] || [];
-
- return [...existingGranules, ...existingNRTGranules];
- };
-
/**
* Query CMR to get dates
* @param {object} def - Layer specs
@@ -127,50 +66,41 @@ export default function granuleLayerBuilder(cache, store, createLayerWMTS) {
*/
const getQueriedGranuleDates = async (def, date) => {
const {
- endDate, startDate, title, visible, dateRanges,
+ title,
} = def;
const state = store.getState();
const { proj: { selected: { crs } } } = state;
const getGranulesUrl = getGranulesUrlSelector(state);
const params = getParamsForGranuleRequest(def, date, crs);
const nrtParams = getParamsForGranuleRequest(def, date, crs, true);
- const { shortName } = params;
- const { shortName: nrtShortName } = nrtParams;
let data = [];
let nrtData = [];
- const mergedExistingGranules = getGranulesFromStore(shortName, nrtShortName);
- const datesQueried = datesHaveBeenQueried(params.startDate, date, mergedExistingGranules);
-
- if (mergedExistingGranules.length && datesQueried) {
- return mergedExistingGranules;
- }
try {
showLoading();
const requestUrl = getGranulesUrl(params);
const nrtRequestUrl = getGranulesUrl(nrtParams);
- const response = await fetch(requestUrl, CMR_AJAX_OPTIONS);
- const nrtResponse = await fetch(nrtRequestUrl, CMR_AJAX_OPTIONS);
- data = await response.json();
- nrtData = await nrtResponse.json();
- data = data.feed.entry;
- nrtData = nrtData.feed.entry;
- if (data.length || nrtData.length) {
- addGranuleCMRDateData(def, nrtData, nrtShortName, dateRanges);
- addGranuleCMRDateData(def, data, shortName, dateRanges);
- } else {
- const dateWithinRange = isWithinDateRange(date, startDate, endDate);
- // only show modal error if layer not set to hidden and outside of selected date range
- if (visible && dateWithinRange) throttleDispathCMRErrorDialog(title);
- }
+ const requests = [fetch(requestUrl, CMR_AJAX_OPTIONS), fetch(nrtRequestUrl, CMR_AJAX_OPTIONS)];
+ const responses = await Promise.allSettled(requests);
+ const fulfilledResponses = responses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);
+ const [response, nrtResponse] = fulfilledResponses;
+ const jsonRequests = [response.json(), nrtResponse.json()];
+ const jsonResponses = await Promise.allSettled(jsonRequests);
+ const [responseJson, nrtResponseJson] = jsonResponses.filter(({ status }) => status === 'fulfilled').map(({ value }) => value);
+ data = responseJson.feed.entry;
+ nrtData = nrtResponseJson.feed.entry;
} catch (e) {
console.error(e);
throttleDispathCMRErrorDialog(title);
- return CMRDataStore[crs][shortName];
} finally {
hideLoading();
}
- return getGranulesFromStore(shortName, nrtShortName);
+ const transformedData = [...data, ...nrtData].map((entry) => {
+ const date = toISOStringSeconds(entry.time_start);
+
+ return transformGranuleData(entry, date, crs);
+ });
+ return transformedData;
};
/**