TagSpaces is an offline, open source, document manager with tagging support
listObjectsV2-to-handle-results-over-1000 -add maxLimit location setting (#1922)
Stanimir3bcf0fd3ce8d
diff --git a/app/AppConfig.ts b/app/AppConfig.tsindex 5fd540a5f..26b02ff09 100644--- a/app/AppConfig.ts+++ b/app/AppConfig.ts@@ -28,5 +28,6 @@ AppConfig.editableFiles = ['sql'];AppConfig.autoSaveInterval = 40000;+AppConfig.maxLoops = 5;export default AppConfig;diff --git a/app/components/dialogs/CreateEditLocationDialog.tsx b/app/components/dialogs/CreateEditLocationDialog.tsxindex c62573792..016ecc9e7 100644--- a/app/components/dialogs/CreateEditLocationDialog.tsx+++ b/app/components/dialogs/CreateEditLocationDialog.tsx@@ -126,6 +126,13 @@ function CreateEditLocationDialog(props: Props) {defaultIndexAge = parseInt(maxIndexAsString, 10);}const [maxIndexAge, setMaxIndexAge] = useState<number>(defaultIndexAge);++ let defaultMaxLoops = AppConfig.maxLoops;+ if (location && location.maxLoops && location.maxLoops > 0) {+ const maxLoopsAsString = location.maxLoops + '';+ defaultMaxLoops = parseInt(maxLoopsAsString, 10);+ }+ const [maxLoops, setMaxLoops] = useState<number>(defaultMaxLoops);const [storeName, setStoreName] = useState<string>(location && location.name ? location.name : '');@@ -221,6 +228,13 @@ function CreateEditLocationDialog(props: Props) {}}+ function changeMaxLoops(event: React.ChangeEvent<HTMLInputElement>) {+ const loops = event.target.value;+ if (loops) {+ setMaxLoops(parseInt(loops, 10));+ }+ }+useEffect(() => {if (!firstRender) {validateObjectStore();@@ -393,6 +407,7 @@ function CreateEditLocationDialog(props: Props) {fullTextIndex,watchForChanges: false,maxIndexAge,+ maxLoops,ignorePatternPaths};}@@ -771,6 +786,55 @@ function CreateEditLocationDialog(props: Props) {</Typography>}/>+ {type === locationType.TYPE_CLOUD && (+ <FormControlLabel+ className={classes.formControl}+ labelPlacement="start"+ style={{ justifyContent: 'space-between' }}+ control={+ <Select+ data-tid="maxLoopsTID"+ name="maxLoops"+ onChange={changeMaxLoops}+ value={maxLoops}+ >+ <MenuItem value="1">+ <span>1000</span>+ </MenuItem>+ <MenuItem value="2">+ <span>2000</span>+ </MenuItem>+ <MenuItem value="5">+ <span>5000</span>+ </MenuItem>+ <MenuItem value="10">+ <span>10000</span>+ </MenuItem>+ <MenuItem value="20">+ <span>20000</span>+ </MenuItem>+ <MenuItem value="50">+ <span>50000</span>+ </MenuItem>+ <MenuItem value="100">+ <span>100000</span>+ </MenuItem>+ <MenuItem value="150">+ <span>150000</span>+ </MenuItem>+ <MenuItem value="200">+ <span>200000</span>+ </MenuItem>+ </Select>+ }+ label={+ <Typography>+ {i18n.t('core:maxLoops')}+ <InfoIcon tooltip={i18n.t('core:maxLoopsHelp')} />+ </Typography>+ }+ />+ )}{AppConfig.useSidecarsForFileTaggingDisableSetting ? (<FormControlLabelclassName={classes.formControl}diff --git a/app/containers/PageNotification.tsx b/app/containers/PageNotification.tsxindex 4e4d2f5c1..0a5a1eb25 100644--- a/app/containers/PageNotification.tsx+++ b/app/containers/PageNotification.tsx@@ -97,7 +97,7 @@ function PageNotification(props: Props) {data-tid={props.notificationStatus.tid}anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }}open={props.notificationStatus.visible}- onClose={props.hideNotifications}+ onClose={() => props.hideNotifications()}autoHideDuration={props.notificationStatus.autohide ? 3000 : undefined}message={props.notificationStatus.text}action={[@@ -106,7 +106,7 @@ function PageNotification(props: Props) {key="close"aria-label={i18n.t('core:closeButton')}color="inherit"- onClick={props.hideNotifications}+ onClick={() => props.hideNotifications()}size="large"><CloseIcon />diff --git a/app/locales/en/core.json b/app/locales/en/core.jsonindex ade6f980c..646e70c78 100644--- a/app/locales/en/core.json+++ b/app/locales/en/core.json@@ -779,5 +779,8 @@"switchToMasonry": "Switch to masonry view","switchToDefaultPerspective": "Switch to default perspective","noImagesInFolder": "This folder does not contain any images",+ "warningDirectoryIsTruncated": "Directory is truncated!",+ "maxLoops": "Limit list objects in a bucket",+ "maxLoopsHelp": "list-objects-v2 returns some or all (up to this limit) of the objects in a bucket.","-": "-"}diff --git a/app/package.json b/app/package.jsonindex eae29ae3f..58e95feeb 100644--- a/app/package.json+++ b/app/package.json@@ -16,7 +16,7 @@"@tagspaces/extensions": "1.0.84"},"nodeDependencies": {- "@tagspaces/tagspaces-ws": "2.1.34",+ "@tagspaces/tagspaces-ws": "2.1.36","@tagspaces/extensions": "1.0.84","sharp": "0.31.3"}diff --git a/app/reducers/app.ts b/app/reducers/app.tsindex fb2af8576..ab02a4c2b 100644--- a/app/reducers/app.ts+++ b/app/reducers/app.ts@@ -1338,7 +1338,12 @@ export const actions = {getState(),getState().app.currentLocationId);-+ const resultsLimit = {+ maxLoops: currentLocation.maxLoops+ ? currentLocation.maxLoops+ : AppConfig.maxLoops,+ IsTruncated: false+ };PlatformIO.listDirectoryPromise(directoryPath,fsEntryMeta &&@@ -1347,9 +1352,19 @@ export const actions = {fsEntryMeta.perspective === PerspectiveIDs.GALLERY)? ['extractThumbPath']: [], // mode,- currentLocation ? currentLocation.ignorePatternPaths : []+ currentLocation ? currentLocation.ignorePatternPaths : [],+ resultsLimit).then(results => {+ if (resultsLimit.IsTruncated) {+ dispatch(+ actions.showNotification(+ i18n.t('warningDirectoryIsTruncated') + ': ' + directoryPath,+ 'error',+ false+ )+ );+ }updateHistory(currentLocation, directoryPath);if (results !== undefined) {// console.debug('app listDirectoryPromise resolved:' + results.length);@@ -1451,7 +1466,7 @@ export const actions = {dispatch(actions.setSelectedEntries(newSelectedEntries));}} */- dispatch(actions.hideNotifications());+ dispatch(actions.hideNotifications(['error']));dispatch(actions.loadDirectorySuccessInt(directoryPath,@@ -2079,7 +2094,18 @@ export const actions = {autohide,tid}),- hideNotifications: () => ({+ hideNotifications: (excludeTypes = []) => (+ dispatch: (action) => void,+ getState: () => any+ ) => {+ const { notificationStatus } = getState().app;+ if (+ !excludeTypes.some(type => type === notificationStatus.notificationType)+ ) {+ dispatch(actions.hideNotificationsInt());+ }+ },+ hideNotificationsInt: () => ({type: types.SET_NOTIFICATION,visible: false,text: null,diff --git a/app/services/platform-facade.ts b/app/services/platform-facade.tsindex 639817ccb..2a4f031d9 100644--- a/app/services/platform-facade.ts+++ b/app/services/platform-facade.ts@@ -221,13 +221,15 @@ export default class PlatformFacade {* @param path* @param mode = ['extractTextContent', 'extractThumbPath']* @param ignorePatterns+ * @param resultsLimit*/static listDirectoryPromise = (path: string,mode = ['extractThumbPath'],- ignorePatterns: Array<string> = []+ ignorePatterns: Array<string> = [],+ resultsLimit: any = {}): Promise<Array<any>> =>- platformListDirectoryPromise(path, mode, ignorePatterns);+ platformListDirectoryPromise(path, mode, ignorePatterns, resultsLimit);static listMetaDirectoryPromise = (path: string): Promise<Array<any>> =>platformListMetaDirectoryPromise(path);diff --git a/app/tagspaces.namespace.d.ts b/app/tagspaces.namespace.d.tsindex 6a4bf5194..6795ba59c 100644--- a/app/tagspaces.namespace.d.ts+++ b/app/tagspaces.namespace.d.ts@@ -43,6 +43,7 @@ export namespace TS {disableIndexing?: boolean;fullTextIndex?: boolean;maxIndexAge?: number;+ maxLoops?: number;persistTagsInSidecarFile?: boolean;ignorePatternPaths?: Array<string>;}diff --git a/package-lock.json b/package-lock.jsonindex 661511c68..45c85bfbf 100644--- a/package-lock.json+++ b/package-lock.json@@ -21,7 +21,7 @@"@mui/styles": "5.10.15","@mui/x-date-pickers": "^5.0.8","@tagspaces/tagspaces-md": "2.1.6",- "@tagspaces/tagspaces-platforms": "2.1.34",+ "@tagspaces/tagspaces-platforms": "2.1.36","aws-amplify": "^4.3.8","aws-sdk": "^2.1036.0","babel-polyfill": "^6.26.0",@@ -12309,9 +12309,9 @@}},"node_modules/@tagspaces/tagspaces-platforms": {- "version": "2.1.34",- "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.34.tgz",- "integrity": "sha512-JcqQHE6ElieSPWuqpJ222ko8fYq3uxraCAz5rgg67KyRXX/qHIquO933wHZjZHGZqRvANFBE0AdunSVPGJS4AQ==",+ "version": "2.1.36",+ "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.36.tgz",+ "integrity": "sha512-MSkpEv51Jiw/ALGeMEORrtgq2ftPDs3y9ELxBLw6+SbMJ4HpVjir5eN2NjN4HCQRcPOt4QFuFpNFmzdM7dLEyw==","hasInstallScript": true},"node_modules/@testim/chrome-version": {@@ -51894,9 +51894,9 @@}},"@tagspaces/tagspaces-platforms": {- "version": "2.1.34",- "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.34.tgz",- "integrity": "sha512-JcqQHE6ElieSPWuqpJ222ko8fYq3uxraCAz5rgg67KyRXX/qHIquO933wHZjZHGZqRvANFBE0AdunSVPGJS4AQ=="+ "version": "2.1.36",+ "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.36.tgz",+ "integrity": "sha512-MSkpEv51Jiw/ALGeMEORrtgq2ftPDs3y9ELxBLw6+SbMJ4HpVjir5eN2NjN4HCQRcPOt4QFuFpNFmzdM7dLEyw=="},"@testim/chrome-version": {"version": "1.1.3",diff --git a/package.json b/package.jsonindex 6d2e65189..dcc921015 100644--- a/package.json+++ b/package.json@@ -150,7 +150,7 @@"@mui/styles": "5.10.15","@mui/x-date-pickers": "^5.0.8","@tagspaces/tagspaces-md": "2.1.6",- "@tagspaces/tagspaces-platforms": "2.1.34",+ "@tagspaces/tagspaces-platforms": "2.1.36","aws-amplify": "^4.3.8","aws-sdk": "^2.1036.0","babel-polyfill": "^6.26.0",