TagSpaces is an offline, open source, document manager with tagging support
TSN842 extract loading of meta sidecar from the loadDirectoryContent (#1604)
Stanimirc38e4651e433
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.ymlindex b241d0925..861c4b5a1 100644--- a/.github/workflows/pull_request.yml+++ b/.github/workflows/pull_request.yml@@ -84,7 +84,7 @@ jobs:- name: Checkout TagspacesProuses: actions/checkout@v2with:- ref: TSN759---Fully-migrate-the-electron-and-cordova-platforms-to-new-the-tagspaces-platform+ ref: developrepository: tagspaces/tagspacesprotoken: ${{ secrets.CHECKOUT_TSPRO }}path: extensions/tagspacesprodiff --git a/app/package.json b/app/package.jsonindex a4faecae5..b94108a54 100644--- a/app/package.json+++ b/app/package.json@@ -13,8 +13,8 @@"license": "AGPL-3.0","scripts": {},"dependencies": {- "@tagspaces/extensions": "1.0.23","@tagspaces/tagspaces-ws": "1.2.1",+ "@tagspaces/extensions": "1.0.23","sharp": "^0.29.3"}}diff --git a/app/perspectives/grid-perspective/components/GridPagination.tsx b/app/perspectives/grid-perspective/components/GridPagination.tsxindex bd9c3276a..ac3ab37eb 100644--- a/app/perspectives/grid-perspective/components/GridPagination.tsx+++ b/app/perspectives/grid-perspective/components/GridPagination.tsx@@ -16,20 +16,30 @@**/-import React, { useState, useEffect, useRef } from 'react';+import React, { useState, useEffect, useRef, useReducer } from 'react';import { connect } from 'react-redux';import Typography from '@material-ui/core/Typography';import Tooltip from '@material-ui/core/Tooltip';import Pagination from '@material-ui/lab/Pagination';+import { bindActionCreators } from 'redux';import i18n from '-/services/i18n';import {+ actions as AppActions,getCurrentDirectoryColor,+ getPageEntries,getSearchResultCount,isLoading} from '-/reducers/app';import EntryIcon from '-/components/EntryIcon';import AppConfig from '-/config';import { TS } from '-/tagspaces.namespace';+import { getMetaForEntry } from '-/services/utils-io';+import {+ getMetaFileLocationForDir,+ getMetaFileLocationForFile,+ getThumbFileLocationForFile+} from '-/utils/paths';+import PlatformIO from '-/services/platform-facade';interface Props {className: string;@@ -39,6 +49,7 @@ interface Props {directories: Array<TS.FileSystemEntry>;showDirectories: boolean;files: Array<TS.FileSystemEntry>;+ pageEntries: Array<TS.FileSystemEntry>;renderCell: (entry: TS.FileSystemEntry, isLast?: boolean) => void;currentDirectoryColor: string;isAppLoading: boolean;@@ -49,6 +60,10 @@ interface Props {searchResultCount: number;onContextMenu: (event: React.MouseEvent<HTMLDivElement>) => void;onClick: (event: React.MouseEvent<HTMLDivElement>) => void;+ updateCurrentDirEntries: (+ dirEntries: TS.FileSystemEntry[],+ pageEntries: any+ ) => void;}const GridPagination = (props: Props) => {@@ -62,17 +77,53 @@ const GridPagination = (props: Props) => {isAppLoading,currentDirectoryColor,gridPageLimit,- currentPage+ currentPage,+ files,+ pageEntries} = props;- let { files } = props;const allFilesCount = files.length;+ const showPagination = gridPageLimit && files.length > gridPageLimit;+ const paginationCount = showPagination+ ? Math.ceil(allFilesCount / gridPageLimit)+ : 10;+const containerEl = useRef(null);- const [page, setPage] = useState(currentPage);+ // const entriesUpdated = useRef([]);+ const page = useRef<number>(currentPage);+ // const [page, setPage] = useState(currentPage);++ let pageFiles;+ if (showPagination) {+ const start = (page.current - 1) * gridPageLimit;+ pageFiles = files.slice(start, start + gridPageLimit);+ } else {+ pageFiles = files;+ }+ const [, forceUpdate] = useReducer(x => x + 1, 0);useEffect(() => {- if (page !== currentPage) {+ // if (PlatformIO.haveObjectStoreSupport()) {+ PlatformIO.listMetaDirectoryPromise(props.currentDirectoryPath)+ .then(meta => {+ const dirEntriesPromises = getDirEntriesPromises();+ const fileEntriesPromises = getFileEntriesPromises(meta);+ const thumbs = getThumbs(meta);+ updateEntries([+ ...dirEntriesPromises,+ ...fileEntriesPromises,+ ...thumbs+ ]);+ return true;+ })+ .catch(ex => console.error(ex));+ // }+ }, [page.current, files]);++ useEffect(() => {+ page.current = currentPage;+ /* if (page !== currentPage) {setPage(props.currentPage);- }+ } */if (containerEl && containerEl.current) {containerEl.current.scrollTop = 0;}@@ -82,24 +133,116 @@ const GridPagination = (props: Props) => {props.searchResultCount]);+ const setThumbs = (+ entry: TS.FileSystemEntry,+ meta: Array<any>+ ): TS.FileSystemEntry => {+ const thumbEntry = { ...entry };+ let thumbPath = getThumbFileLocationForFile(entry.path, '/', false);+ if (meta.some(metaFile => thumbPath.endsWith(metaFile.path))) {+ thumbEntry.thumbPath = thumbPath;+ if (PlatformIO.haveObjectStoreSupport()) {+ if (thumbPath && thumbPath.startsWith('/')) {+ thumbPath = thumbPath.substring(1);+ }++ thumbPath = PlatformIO.getURLforPath(thumbPath, 604800);+ if (thumbPath) {+ thumbEntry.thumbPath = thumbPath;+ }+ }+ }+ return thumbEntry;+ };++ const getThumbs = (meta: Array<any>): Promise<any>[] =>+ pageFiles.map(entry =>+ Promise.resolve({ [entry.path]: setThumbs(entry, meta) })+ );++ const getDirEntriesPromises = (): Promise<any>[] =>+ directories.map(entry => {+ const metaFilePath = getMetaFileLocationForDir(+ entry.path,+ PlatformIO.getDirSeparator()+ );+ if (+ // meta.some(metaFile => metaFilePath.endsWith(metaFile)) &&+ !checkEntryExist(entry.path) &&+ entry.path.indexOf(+ AppConfig.metaFolder + PlatformIO.getDirSeparator()+ ) === -1+ ) {+ return getMetaForEntry(entry, metaFilePath);+ }+ return Promise.resolve({ [entry.path]: undefined });+ });++ const getFileEntriesPromises = (meta: Array<any>): Promise<any>[] =>+ pageFiles.map(entry => {+ const metaFilePath = getMetaFileLocationForFile(+ entry.path,+ PlatformIO.getDirSeparator()+ );+ if (+ // check if metaFilePath exist in listMetaDirectory content+ meta.some(metaFile => metaFilePath.endsWith(metaFile.path)) &&+ !checkEntryExist(entry.path) &&+ entry.path.indexOf(+ AppConfig.metaFolder + PlatformIO.getDirSeparator()+ ) === -1+ ) {+ return getMetaForEntry(entry, metaFilePath);+ }+ return Promise.resolve({ [entry.path]: undefined });+ });++ const updateEntries = metaPromises => {+ const catchHandler = error => undefined;+ Promise.all(metaPromises.map(promise => promise.catch(catchHandler)))+ .then(entries => {+ updateCurrentDirEntries(entries.filter(entry => entry !== undefined));+ // entriesUpdated.current = entries;+ return true;+ })+ .catch(err => {+ console.error('err updateEntries:', err);+ });+ };++ const updateCurrentDirEntries = entries => {+ const entriesEnhanced = [];+ entries.forEach(entry => {+ for (const [key, value] of Object.entries(entry)) {+ if (value) {+ // !checkEntryExist(key)) {+ entriesEnhanced.push(value);+ }+ }+ });+ if (entriesEnhanced.length > 0) {+ props.updateCurrentDirEntries(entriesEnhanced, entries);+ }+ };++ const checkEntryExist = path => {+ const index = pageEntries.findIndex(+ objUpdated => Object.keys(objUpdated).indexOf(path) > -1+ );+ return index > -1;+ };+const handleChange = (event, value) => {- setPage(value);+ // setPage(value);+ page.current = value;+ forceUpdate();if (containerEl && containerEl.current) {containerEl.current.scrollTop = 0;}};- let paginationCount = 10;-- let showPagination = false;- if (gridPageLimit && files.length > gridPageLimit) {- paginationCount = Math.ceil(files.length / gridPageLimit);- const start = (page - 1) * gridPageLimit;- files = files.slice(start, start + gridPageLimit);- showPagination = true;- }-return (+ // eslint-disable-next-line jsx-a11y/click-events-have-key-events,jsx-a11y/no-noninteractive-element-interactions,jsx-a11y/no-static-element-interactions<divref={containerEl}onContextMenu={(event: React.MouseEvent<HTMLDivElement>) =>@@ -120,11 +263,11 @@ const GridPagination = (props: Props) => {style={style}data-tid="perspectiveGridFileTable">- {page === 1 && directories.map(entry => renderCell(entry))}- {files.map((entry, index, dArray) =>+ {page.current === 1 && directories.map(entry => renderCell(entry))}+ {pageFiles.map((entry, index, dArray) =>renderCell(entry, index === dArray.length - 1))}- {!isAppLoading && files.length < 1 && directories.length < 1 && (+ {!isAppLoading && pageFiles.length < 1 && directories.length < 1 && (<div style={{ textAlign: 'center' }}><EntryIcon isFile={false} /><Typography@@ -138,7 +281,7 @@ const GridPagination = (props: Props) => {</div>)}{!isAppLoading &&- files.length < 1 &&+ pageFiles.length < 1 &&directories.length >= 1 &&!showDirectories && (<div style={{ textAlign: 'center' }}>@@ -176,12 +319,12 @@ const GridPagination = (props: Props) => {padding: 3}}count={paginationCount}- page={page}+ page={page.current}onChange={handleChange}/></Tooltip>)}- {!showPagination && (directories.length > 0 || files.length > 0) && (+ {!showPagination && (directories.length > 0 || pageFiles.length > 0) && (<div style={{ padding: 15, bottom: 10 }}><Typographystyle={{@@ -204,8 +347,25 @@ function mapStateToProps(state) {return {isAppLoading: isLoading(state),currentDirectoryColor: getCurrentDirectoryColor(state),- searchResultCount: getSearchResultCount(state)+ searchResultCount: getSearchResultCount(state),+ pageEntries: getPageEntries(state)};}-export default connect(mapStateToProps)(GridPagination);+function mapActionCreatorsToProps(dispatch) {+ return bindActionCreators(+ {+ updateCurrentDirEntries: AppActions.updateCurrentDirEntries+ },+ dispatch+ );+}++const areEqual = (prevProp: Props, nextProp: Props) =>+ JSON.stringify(nextProp.files) === JSON.stringify(prevProp.files) &&+ JSON.stringify(nextProp.directories) === JSON.stringify(prevProp.directories);++export default connect(+ mapStateToProps,+ mapActionCreatorsToProps+)(React.memo(GridPagination, areEqual));diff --git a/app/reducers/app.ts b/app/reducers/app.tsindex 4de599698..2bb0f465a 100644--- a/app/reducers/app.ts+++ b/app/reducers/app.ts@@ -73,6 +73,7 @@ export const types = {LOAD_DIRECTORY_SUCCESS: 'APP/LOAD_DIRECTORY_SUCCESS',LOAD_DIRECTORY_FAILURE: 'APP/LOAD_DIRECTORY_FAILURE',CLEAR_DIRECTORY_CONTENT: 'APP/CLEAR_DIRECTORY_CONTENT',+ // LOAD_PAGE_CONTENT: 'APP/LOAD_PAGE_CONTENT',SET_SEARCH_RESULTS: 'APP/SET_SEARCH_RESULTS',APPEND_SEARCH_RESULTS: 'APP/APPEND_SEARCH_RESULTS',OPEN_FILE: 'APP/OPEN_FILE',@@ -119,6 +120,7 @@ export const types = {// REFLECT_UPDATE_SIDECARTAGS: 'APP/REFLECT_UPDATE_SIDECARTAGS',// REFLECT_UPDATE_SIDECARMETA: 'APP/REFLECT_UPDATE_SIDECARMETA',UPDATE_CURRENTDIR_ENTRY: 'APP/UPDATE_CURRENTDIR_ENTRY',+ UPDATE_CURRENTDIR_ENTRIES: 'APP/UPDATE_CURRENTDIR_ENTRIES',SET_ISLOADING: 'APP/SET_ISLOADING'};@@ -270,6 +272,7 @@ export default (state: any = initialState, action: any) => {return {...state,currentDirectoryEntries: action.directoryContent,+ pageEntries: [],currentDirectoryColor: action.directoryMeta? action.directoryMeta.color || '': '',@@ -306,6 +309,21 @@ export default (state: any = initialState, action: any) => {currentDirectoryPath: ''};}+ /* case types.LOAD_PAGE_CONTENT: {+ const newPageEntries = [...state.pageEntries];+ for (let i = 0; i < action.pageEntries.length; i += 1) {+ const index = newPageEntries.findIndex(+ entry => entry.path === action.pageEntries[i].path+ );+ if (index === -1) {+ newPageEntries.push(action.pageEntries[i]);+ }+ }+ return {+ ...state,+ pageEntries: newPageEntries+ };+ } */case types.SET_CURRENLOCATIONID: {return {...state,@@ -623,6 +641,33 @@ export default (state: any = initialState, action: any) => {})};} */+ case types.UPDATE_CURRENTDIR_ENTRIES: {+ const newDirEntries = [...state.currentDirectoryEntries];+ for (let i = 0; i < newDirEntries.length; i += 1) {+ const dirEntry = action.dirEntries.find(+ entry => entry.path === newDirEntries[i].path+ );+ if (dirEntry) {+ newDirEntries[i] = dirEntry;+ }+ }++ /* const newPageEntries = [...state.pageEntries];+ for (const [path, value] of Object.entries(newPageEntries)) {+ const pageEntry = action.pageEntries.findIndex(+ entry => entry.path === newPageEntries[i].path+ );+ if (pageEntry) {+ newPageEntries[i] = pageEntry;+ }+ } */++ return {+ ...state,+ currentDirectoryEntries: newDirEntries,+ pageEntries: action.pageEntries+ };+ }case types.UPDATE_CURRENTDIR_ENTRY: {return {...state,@@ -968,9 +1013,12 @@ export const actions = {password: '1234',port: 8080}); */+ /* const mode = PlatformIO.haveObjectStoreSupport()+ ? []+ : ['extractThumbPath', 'extractThumbURL']; */PlatformIO.listDirectoryPromise(directoryPath,- ['extractThumbPath', 'extractThumbURL'],+ [], // mode,currentLocation ? currentLocation.ignorePatternPaths : []).then(results => {@@ -998,7 +1046,8 @@ export const actions = {},loadDirectoryContent: (directoryPath: string,- generateThumbnails: boolean+ generateThumbnails: boolean,+ loadDirMeta: boolean = false) => async (dispatch: (actions: Object) => void, getState: () => any) => {// console.debug('loadDirectoryContent:' + directoryPath);window.walkCanceled = false;@@ -1010,28 +1059,34 @@ export const actions = {if (selectedEntries.length > 0) {dispatch(actions.setSelectedEntries([]));}- try {- const fsEntryMeta = await loadMetaDataPromise(- normalizePath(directoryPath) + PlatformIO.getDirSeparator()- );- // console.debug('Loading meta succeeded for:' + directoryPath);- dispatch(- actions.loadDirectoryContentInt(- directoryPath,- generateThumbnails,- fsEntryMeta- )- );- /* if (fsEntryMeta.color) { // TODO rethink this states changes are expensive- dispatch(actions.setCurrentDirectoryColor(fsEntryMeta.color));- }- if (fsEntryMeta.perspective) {- dispatch(actions.setCurrentDirPerspective(fsEntryMeta.perspective));- } */+ if (loadDirMeta) {+ try {+ const fsEntryMeta = await loadMetaDataPromise(+ normalizePath(directoryPath) + PlatformIO.getDirSeparator()+ );+ // console.debug('Loading meta succeeded for:' + directoryPath);+ dispatch(+ actions.loadDirectoryContentInt(+ directoryPath,+ generateThumbnails,+ fsEntryMeta+ )+ );+ /* if (fsEntryMeta.color) { // TODO rethink this states changes are expensive+ dispatch(actions.setCurrentDirectoryColor(fsEntryMeta.color));+ }+ if (fsEntryMeta.perspective) {+ dispatch(actions.setCurrentDirPerspective(fsEntryMeta.perspective));+ } */- // return true;- } catch (err) {- console.debug('Error loading meta of:' + directoryPath + ' ' + err);+ // return true;+ } catch (err) {+ console.debug('Error loading meta of:' + directoryPath + ' ' + err);+ dispatch(+ actions.loadDirectoryContentInt(directoryPath, generateThumbnails)+ );+ }+ } else {dispatch(actions.loadDirectoryContentInt(directoryPath, generateThumbnails));@@ -1841,6 +1896,14 @@ export const actions = {path,entry}),+ updateCurrentDirEntries: (+ dirEntries: TS.FileSystemEntry[],+ pageEntries: any+ ) => ({+ type: types.UPDATE_CURRENTDIR_ENTRIES,+ dirEntries,+ pageEntries+ }),/*** @param path* @param tags@@ -2189,6 +2252,7 @@ export const actions = {export const currentUser = (state: any) => state.app.user;export const getDirectoryContent = (state: any) =>state.app.currentDirectoryEntries;+export const getPageEntries = (state: any) => state.app.pageEntries;export const getCurrentDirectoryFiles = (state: any) =>state.app.currentDirectoryFiles;export const getCurrentDirectoryDirs = (state: any) =>diff --git a/app/services/platform-facade.ts b/app/services/platform-facade.tsindex 35fca5d89..4054a4821 100644--- a/app/services/platform-facade.ts+++ b/app/services/platform-facade.ts@@ -38,6 +38,7 @@ import {platformCreateThumbnailsInWorker,platformGetURLforPath,platformListDirectoryPromise,+ platformListMetaDirectoryPromise,platformListObjectStoreDir,platformSaveFilePromise,platformGetPropertiesPromise,@@ -175,6 +176,9 @@ export default class PlatformFacade {): Promise<Array<any>> =>platformListDirectoryPromise(path, mode, ignorePatterns);+ static listMetaDirectoryPromise = (path: string): Promise<Array<any>> =>+ platformListMetaDirectoryPromise(path);+static listObjectStoreDir = (param: Object,mode = ['extractThumbPath'],diff --git a/app/services/utils-io.ts b/app/services/utils-io.tsindex 1ac0c826b..ca233895b 100644--- a/app/services/utils-io.ts+++ b/app/services/utils-io.ts@@ -139,6 +139,22 @@ export function enhanceDirectoryContent(};}+export async function getMetaForEntry(+ entry: TS.FileSystemEntry,+ metaFilePath: string+): Promise<any> {+ const meta: TS.FileSystemEntryMeta = await loadJSONFile(metaFilePath);+ if (meta) {+ const entryEnhanced = enhanceEntry({ ...entry, meta });+ return { [entry.path]: entryEnhanced };+ }+ return Promise.resolve({ [entry.path]: undefined });+}++/**+ * TODO enhance only entries from the current page+ * @param entry+ */export function enhanceEntry(entry: any): TS.FileSystemEntry {let fileNameTags = [];if (entry.isFile) {@@ -193,7 +209,7 @@ export function enhanceEntry(entry: any): TS.FileSystemEntry {if (sidecarPerspective) {enhancedEntry.perspective = sidecarPerspective;}- // console.log('Enhancing ' + entry.path); console.log(enhancedEntry);+ // console.log('Enhancing ' + entry.path + ':' + JSON.stringify(enhancedEntry));return enhancedEntry;}diff --git a/app/utils/paths.ts b/app/utils/paths.tsindex fecc06d65..866c681f9 100644--- a/app/utils/paths.ts+++ b/app/utils/paths.ts@@ -158,7 +158,11 @@ export function getMetaFileLocationForDir(metaFile: string = AppConfig.metaFolderFile) {const metaFolder = getMetaDirectoryPath(entryPath, dirSeparator);- return metaFolder + dirSeparator + metaFile;+ return (+ metaFolder ++ (metaFolder.endsWith(dirSeparator) ? '' : dirSeparator) ++ metaFile+ );}export function extractFileName(diff --git a/package.json b/package.jsonindex 9f9b7120c..c985880d7 100644--- a/package.json+++ b/package.json@@ -148,7 +148,7 @@"@material-ui/icons": "^4.11.2","@material-ui/lab": "^4.0.0-alpha.60","@material-ui/pickers": "^3.3.10",- "@tagspaces/tagspaces-platforms": "1.3.9",+ "@tagspaces/tagspaces-platforms": "1.3.16","aws-amplify": "^4.3.8","aws-sdk": "^2.1036.0","babel-polyfill": "^6.26.0",@@ -159,7 +159,7 @@"coordinate-parser": "^1.0.7","core-js": "3.19.1","crypto-js": "^4.1.1",- "csv-stringify": "^6.0.5",+ "csv-stringify": "^6.1.0","date-fns": "^2.26.0","dompurify": "^2.3.3","electron-debug": "^3.2.0",diff --git a/yarn.lock b/yarn.lockindex 69094c0fa..6073c5d33 100644--- a/yarn.lock+++ b/yarn.lock@@ -5816,10 +5816,10 @@dependencies:defer-to-connect "^1.0.1"-"@tagspaces/[email protected]":- version "1.3.9"- resolved "https://registry.yarnpkg.com/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-1.3.9.tgz#01b19b75921acf55e5f6a189f944d6b87a9b11cc"- integrity sha512-/s6Z2vpmLo2Gak/4aTQuRCD0hnGEH/11KghN/d0vRslBiAWE8Z/Nz0aeLo0I9DFlo3YQZ5YCMGi+tqz4osdhgw==+"@tagspaces/[email protected]":+ version "1.3.16"+ resolved "https://registry.yarnpkg.com/@tagspaces/tagspaces-platforms/-/tagspaces-platforms-1.3.16.tgz#8ac72fe81e6a9979b66b200e8dc617cbc3b84c60"+ integrity sha512-VtNCfJeGMX+RfCmPYQGcQq66cnxA5i/bt13jy4ljYO80zixocauPHL2meWqY7qI0l9eI5Gw8Cyf87jUn614Gjg=="@testim/chrome-version@^1.0.7":version "1.0.7"@@ -9463,10 +9463,10 @@ csstype@^3.0.2:resolved "https://registry.npmjs.org/csstype/-/csstype-3.0.3.tgz"integrity sha512-jPl+wbWPOWJ7SXsWyqGRk3lGecbar0Cb0OvZF/r/ZU011R4YqiRehgkQ9p4eQfo9DSDLqLL3wHwfxeJiuIsNag==-csv-stringify@^6.0.5:- version "6.0.5"- resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.0.5.tgz#3474a4fe784249eb5c91d455f616e1f70961cdc0"- integrity sha512-7xpV3uweJCFF/Ssn56l3xsR/k2r3UqszwjEhej9qEn2cCPzyK1WyHCgoUVzBA792x8HbwonNX7CU9XM2K5s5yw==+csv-stringify@^6.1.0:+ version "6.1.0"+ resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.1.0.tgz#6b79e2717bc35ae063800a490244117998a2e86c"+ integrity sha512-rdBqiy77TczvhGlpLHyoph2adMs6WMnmQY4PBRqeIWykI1FLnVCppnRdso8faGj2+eN7izk9YlbFyZkOrL9rAQ==current-git-branch@^1.1.0:version "1.1.0"