Skip to content

TagSpaces is an offline, open source, document manager with tagging support

listObjectsV2-to-handle-results-over-1000 -add maxLimit location setting (#1922)

Stanimir3bcf0fd3ce8d
diff --git a/app/AppConfig.ts b/app/AppConfig.ts
index 5fd540a5f..26b02ff09 100644
--- a/app/AppConfig.ts
+++ b/app/AppConfig.ts
@@ -28,5 +28,6 @@ AppConfig.editableFiles = [
'sql'
];
AppConfig.autoSaveInterval = 40000;
+AppConfig.maxLoops = 5;
export default AppConfig;
diff --git a/app/components/dialogs/CreateEditLocationDialog.tsx b/app/components/dialogs/CreateEditLocationDialog.tsx
index c62573792..016ecc9e7 100644
--- a/app/components/dialogs/CreateEditLocationDialog.tsx
+++ b/app/components/dialogs/CreateEditLocationDialog.tsx
@@ -126,6 +126,13 @@ function CreateEditLocationDialog(props: Props) {
defaultIndexAge = parseInt(maxIndexAsString, 10);
}
const [maxIndexAge, setMaxIndexAge] = useState<number>(defaultIndexAge);
+
+ let defaultMaxLoops = AppConfig.maxLoops;
+ if (location && location.maxLoops && location.maxLoops > 0) {
+ const maxLoopsAsString = location.maxLoops + '';
+ defaultMaxLoops = parseInt(maxLoopsAsString, 10);
+ }
+ const [maxLoops, setMaxLoops] = useState<number>(defaultMaxLoops);
const [storeName, setStoreName] = useState<string>(
location && location.name ? location.name : ''
);
@@ -221,6 +228,13 @@ function CreateEditLocationDialog(props: Props) {
}
}
+ function changeMaxLoops(event: React.ChangeEvent<HTMLInputElement>) {
+ const loops = event.target.value;
+ if (loops) {
+ setMaxLoops(parseInt(loops, 10));
+ }
+ }
+
useEffect(() => {
if (!firstRender) {
validateObjectStore();
@@ -393,6 +407,7 @@ function CreateEditLocationDialog(props: Props) {
fullTextIndex,
watchForChanges: false,
maxIndexAge,
+ maxLoops,
ignorePatternPaths
};
}
@@ -771,6 +786,55 @@ function CreateEditLocationDialog(props: Props) {
</Typography>
}
/>
+ {type === locationType.TYPE_CLOUD && (
+ <FormControlLabel
+ className={classes.formControl}
+ labelPlacement="start"
+ style={{ justifyContent: 'space-between' }}
+ control={
+ <Select
+ data-tid="maxLoopsTID"
+ name="maxLoops"
+ onChange={changeMaxLoops}
+ value={maxLoops}
+ >
+ <MenuItem value="1">
+ <span>1000</span>
+ </MenuItem>
+ <MenuItem value="2">
+ <span>2000</span>
+ </MenuItem>
+ <MenuItem value="5">
+ <span>5000</span>
+ </MenuItem>
+ <MenuItem value="10">
+ <span>10000</span>
+ </MenuItem>
+ <MenuItem value="20">
+ <span>20000</span>
+ </MenuItem>
+ <MenuItem value="50">
+ <span>50000</span>
+ </MenuItem>
+ <MenuItem value="100">
+ <span>100000</span>
+ </MenuItem>
+ <MenuItem value="150">
+ <span>150000</span>
+ </MenuItem>
+ <MenuItem value="200">
+ <span>200000</span>
+ </MenuItem>
+ </Select>
+ }
+ label={
+ <Typography>
+ {i18n.t('core:maxLoops')}
+ <InfoIcon tooltip={i18n.t('core:maxLoopsHelp')} />
+ </Typography>
+ }
+ />
+ )}
{AppConfig.useSidecarsForFileTaggingDisableSetting ? (
<FormControlLabel
className={classes.formControl}
diff --git a/app/containers/PageNotification.tsx b/app/containers/PageNotification.tsx
index 4e4d2f5c1..0a5a1eb25 100644
--- a/app/containers/PageNotification.tsx
+++ b/app/containers/PageNotification.tsx
@@ -97,7 +97,7 @@ function PageNotification(props: Props) {
data-tid={props.notificationStatus.tid}
anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }}
open={props.notificationStatus.visible}
- onClose={props.hideNotifications}
+ onClose={() => props.hideNotifications()}
autoHideDuration={props.notificationStatus.autohide ? 3000 : undefined}
message={props.notificationStatus.text}
action={[
@@ -106,7 +106,7 @@ function PageNotification(props: Props) {
key="close"
aria-label={i18n.t('core:closeButton')}
color="inherit"
- onClick={props.hideNotifications}
+ onClick={() => props.hideNotifications()}
size="large"
>
<CloseIcon />
diff --git a/app/locales/en/core.json b/app/locales/en/core.json
index ade6f980c..646e70c78 100644
--- a/app/locales/en/core.json
+++ b/app/locales/en/core.json
@@ -779,5 +779,8 @@
"switchToMasonry": "Switch to masonry view",
"switchToDefaultPerspective": "Switch to default perspective",
"noImagesInFolder": "This folder does not contain any images",
+ "warningDirectoryIsTruncated": "Directory is truncated!",
+ "maxLoops": "Limit list objects in a bucket",
+ "maxLoopsHelp": "list-objects-v2 returns some or all (up to this limit) of the objects in a bucket.",
"-": "-"
}
diff --git a/app/package.json b/app/package.json
index eae29ae3f..58e95feeb 100644
--- a/app/package.json
+++ b/app/package.json
@@ -16,7 +16,7 @@
"@tagspaces/extensions": "1.0.84"
},
"nodeDependencies": {
- "@tagspaces/tagspaces-ws": "2.1.34",
+ "@tagspaces/tagspaces-ws": "2.1.36",
"@tagspaces/extensions": "1.0.84",
"sharp": "0.31.3"
}
diff --git a/app/reducers/app.ts b/app/reducers/app.ts
index fb2af8576..ab02a4c2b 100644
--- a/app/reducers/app.ts
+++ b/app/reducers/app.ts
@@ -1338,7 +1338,12 @@ export const actions = {
getState(),
getState().app.currentLocationId
);
-
+ const resultsLimit = {
+ maxLoops: currentLocation.maxLoops
+ ? currentLocation.maxLoops
+ : AppConfig.maxLoops,
+ IsTruncated: false
+ };
PlatformIO.listDirectoryPromise(
directoryPath,
fsEntryMeta &&
@@ -1347,9 +1352,19 @@ export const actions = {
fsEntryMeta.perspective === PerspectiveIDs.GALLERY)
? ['extractThumbPath']
: [], // mode,
- currentLocation ? currentLocation.ignorePatternPaths : []
+ currentLocation ? currentLocation.ignorePatternPaths : [],
+ resultsLimit
)
.then(results => {
+ if (resultsLimit.IsTruncated) {
+ dispatch(
+ actions.showNotification(
+ i18n.t('warningDirectoryIsTruncated') + ': ' + directoryPath,
+ 'error',
+ false
+ )
+ );
+ }
updateHistory(currentLocation, directoryPath);
if (results !== undefined) {
// console.debug('app listDirectoryPromise resolved:' + results.length);
@@ -1451,7 +1466,7 @@ export const actions = {
dispatch(actions.setSelectedEntries(newSelectedEntries));
}
} */
- dispatch(actions.hideNotifications());
+ dispatch(actions.hideNotifications(['error']));
dispatch(
actions.loadDirectorySuccessInt(
directoryPath,
@@ -2079,7 +2094,18 @@ export const actions = {
autohide,
tid
}),
- hideNotifications: () => ({
+ hideNotifications: (excludeTypes = []) => (
+ dispatch: (action) => void,
+ getState: () => any
+ ) => {
+ const { notificationStatus } = getState().app;
+ if (
+ !excludeTypes.some(type => type === notificationStatus.notificationType)
+ ) {
+ dispatch(actions.hideNotificationsInt());
+ }
+ },
+ hideNotificationsInt: () => ({
type: types.SET_NOTIFICATION,
visible: false,
text: null,
diff --git a/app/services/platform-facade.ts b/app/services/platform-facade.ts
index 639817ccb..2a4f031d9 100644
--- a/app/services/platform-facade.ts
+++ b/app/services/platform-facade.ts
@@ -221,13 +221,15 @@ export default class PlatformFacade {
* @param path
* @param mode = ['extractTextContent', 'extractThumbPath']
* @param ignorePatterns
+ * @param resultsLimit
*/
static listDirectoryPromise = (
path: string,
mode = ['extractThumbPath'],
- ignorePatterns: Array<string> = []
+ ignorePatterns: Array<string> = [],
+ resultsLimit: any = {}
): Promise<Array<any>> =>
- platformListDirectoryPromise(path, mode, ignorePatterns);
+ platformListDirectoryPromise(path, mode, ignorePatterns, resultsLimit);
static listMetaDirectoryPromise = (path: string): Promise<Array<any>> =>
platformListMetaDirectoryPromise(path);
diff --git a/app/tagspaces.namespace.d.ts b/app/tagspaces.namespace.d.ts
index 6a4bf5194..6795ba59c 100644
--- a/app/tagspaces.namespace.d.ts
+++ b/app/tagspaces.namespace.d.ts
@@ -43,6 +43,7 @@ export namespace TS {
disableIndexing?: boolean;
fullTextIndex?: boolean;
maxIndexAge?: number;
+ maxLoops?: number;
persistTagsInSidecarFile?: boolean;
ignorePatternPaths?: Array<string>;
}
diff --git a/package-lock.json b/package-lock.json
index 661511c68..45c85bfbf 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -21,7 +21,7 @@
"@mui/styles": "5.10.15",
"@mui/x-date-pickers": "^5.0.8",
"@tagspaces/tagspaces-md": "2.1.6",
- "@tagspaces/tagspaces-platforms": "2.1.34",
+ "@tagspaces/tagspaces-platforms": "2.1.36",
"aws-amplify": "^4.3.8",
"aws-sdk": "^2.1036.0",
"babel-polyfill": "^6.26.0",
@@ -12309,9 +12309,9 @@
}
},
"node_modules/@tagspaces/tagspaces-platforms": {
- "version": "2.1.34",
- "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.34.tgz",
- "integrity": "sha512-JcqQHE6ElieSPWuqpJ222ko8fYq3uxraCAz5rgg67KyRXX/qHIquO933wHZjZHGZqRvANFBE0AdunSVPGJS4AQ==",
+ "version": "2.1.36",
+ "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.36.tgz",
+ "integrity": "sha512-MSkpEv51Jiw/ALGeMEORrtgq2ftPDs3y9ELxBLw6+SbMJ4HpVjir5eN2NjN4HCQRcPOt4QFuFpNFmzdM7dLEyw==",
"hasInstallScript": true
},
"node_modules/@testim/chrome-version": {
@@ -51894,9 +51894,9 @@
}
},
"@tagspaces/tagspaces-platforms": {
- "version": "2.1.34",
- "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.34.tgz",
- "integrity": "sha512-JcqQHE6ElieSPWuqpJ222ko8fYq3uxraCAz5rgg67KyRXX/qHIquO933wHZjZHGZqRvANFBE0AdunSVPGJS4AQ=="
+ "version": "2.1.36",
+ "resolved": "https://registry.npmjs.org/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-2.1.36.tgz",
+ "integrity": "sha512-MSkpEv51Jiw/ALGeMEORrtgq2ftPDs3y9ELxBLw6+SbMJ4HpVjir5eN2NjN4HCQRcPOt4QFuFpNFmzdM7dLEyw=="
},
"@testim/chrome-version": {
"version": "1.1.3",
diff --git a/package.json b/package.json
index 6d2e65189..dcc921015 100644
--- a/package.json
+++ b/package.json
@@ -150,7 +150,7 @@
"@mui/styles": "5.10.15",
"@mui/x-date-pickers": "^5.0.8",
"@tagspaces/tagspaces-md": "2.1.6",
- "@tagspaces/tagspaces-platforms": "2.1.34",
+ "@tagspaces/tagspaces-platforms": "2.1.36",
"aws-amplify": "^4.3.8",
"aws-sdk": "^2.1036.0",
"babel-polyfill": "^6.26.0",