TagSpaces is an offline, open source, document manager with tagging support
chore: refactoring the index creation
Ilian Sapundshiev3e4b08188e12
diff --git a/src/renderer/hooks/LocationIndexContextProvider.tsx b/src/renderer/hooks/LocationIndexContextProvider.tsxindex fc7b1b591..b61120f38 100644--- a/src/renderer/hooks/LocationIndexContextProvider.tsx+++ b/src/renderer/hooks/LocationIndexContextProvider.tsx@@ -29,6 +29,8 @@ import {executePromisesInBatches,isWorkerAvailable,} from '-/services/utils-io';+import { prepareIndex, fuseOptions } from '@tagspaces/tagspaces-search';+import Fuse from 'fuse.js';import { TS } from '-/tagspaces.namespace';import { CommonLocation } from '-/utils/CommonLocation';import useFirstRender from '-/utils/useFirstRender';@@ -43,7 +45,12 @@ import {import { loadJSONString } from '@tagspaces/tagspaces-common/utils-io';import {createIndex,+ createIncrementalIndex,getMetaIndexFilePath,+ getMetaFullTextFilePath,+ parseFullTextJsonl,+ serializeFullTextJsonl,+ mergeFullTextIntoIndex,} from '@tagspaces/tagspaces-indexer';import React, { createContext, useEffect, useReducer, useRef } from 'react';import { useTranslation } from 'react-i18next';@@ -134,6 +141,10 @@ export const LocationIndexContextProvider = ({const firstRender = useFirstRender();const maxIndexAge = useRef<number>(getMaxIndexAge(currentLocation));const prevLocationId = useRef<string>(currentLocationId);+ const enhancedIndex = useRef<any[]>(undefined);+ const fuseInstance = useRef<any>(undefined);+ const fullTextMap = useRef<Record<string, string>>(undefined);+ const fullTextLoaded = useRef<boolean>(false);useEffect(() => {if (currentLocationId) {@@ -172,6 +183,10 @@ export const LocationIndexContextProvider = ({}function setIndex(i, location: CommonLocation = undefined) {index.current = i;+ enhancedIndex.current = undefined;+ fuseInstance.current = undefined;+ fullTextMap.current = undefined;+ fullTextLoaded.current = false;if (index.current && index.current.length > 0) {indexLoadedOn.current = new Date().getTime();} else {@@ -410,13 +425,14 @@ export const LocationIndexContextProvider = ({return Promise.resolve(undefined);}- function createNotWorkerIndex(+ async function createNotWorkerIndex(param: any,loc: CommonLocation,extractText = false,ignorePatterns: Array<string> = [],isWalking = () => true,extractLinks?: boolean,+ forceFullReindex = false,): Promise<TS.FileSystemEntry[]> {const mode = ['loadMeta'];if (extractText) {@@ -425,34 +441,51 @@ export const LocationIndexContextProvider = ({mode.push('extractLinks');}}- return createIndex(- {- ...param,- listDirectoryPromise: loc.listDirectoryPromise,- getFileContentPromise: loc.getFileContentPromise,- },- mode,- ignorePatterns,- isWalking,- )- .then((directoryIndex) => {- if (!loc.isReadOnly) {- persistIndex(param, directoryIndex).then((success) => {- if (success) {- console.log('Index generated in folder: ' + param.path);- }- });++ const indexParam = {+ ...param,+ listDirectoryPromise: loc.listDirectoryPromise,+ getFileContentPromise: loc.getFileContentPromise,+ };++ // Try incremental indexing if an existing index is available+ let existingIdx = index.current;+ if (!existingIdx && !forceFullReindex) {+ existingIdx = await loadIndexFromDisk(param.path, param.locationID);+ }++ let directoryIndex;+ if (existingIdx && existingIdx.length > 0 && !forceFullReindex) {+ console.log('Attempting incremental index for: ' + param.path);+ const result = await createIncrementalIndex(+ indexParam,+ mode,+ ignorePatterns,+ isWalking,+ existingIdx,+ fullTextMap.current,+ );+ directoryIndex = result.index;+ console.log(+ `Incremental index stats: +${result.stats.added} ~${result.stats.modified} -${result.stats.deleted} =${result.stats.unchanged}`,+ );+ } else {+ directoryIndex = await createIndex(+ indexParam,+ mode,+ ignorePatterns,+ isWalking,+ );+ }++ if (!loc.isReadOnly) {+ persistIndex(param, directoryIndex).then((success) => {+ if (success) {+ console.log('Index generated in folder: ' + param.path);}- return enhanceDirectoryIndex(- directoryIndex,- param.locationID,- param.path,- );- })- .catch((err) => {- console.log('Error creating index: ', err);- return undefined;});+ }+ return enhanceDirectoryIndex(directoryIndex, param.locationID, param.path);}function createDirectoryIndexWrapper(@@ -626,16 +659,110 @@ export const LocationIndexContextProvider = ({});}- function getSearchResults(+ function getOrCreateEnhancedIndex(+ searchIndex: TS.FileSystemEntry[],+ showUnixHiddenEntries: boolean,+ ) {+ // Use cached enhanced index if the raw index hasn't changed+ if (enhancedIndex.current && searchIndex === index.current) {+ return enhancedIndex.current;+ }+ const prepared = prepareIndex(+ searchIndex,+ tagDelimiter,+ showUnixHiddenEntries,+ );+ // Only cache if this is the current location's index+ if (searchIndex === index.current) {+ enhancedIndex.current = prepared;+ }+ return prepared;+ }++ function getOrCreateFuse(entries: any[]) {+ if (fuseInstance.current) {+ return fuseInstance.current;+ }+ fuseInstance.current = new Fuse(entries, fuseOptions);+ return fuseInstance.current;+ }++ async function loadFullTextIfNeeded(searchIndex: TS.FileSystemEntry[]) {+ // Only load fulltext for the current location's index, and only once+ if (+ fullTextLoaded.current ||+ searchIndex !== index.current ||+ !currentLocation+ ) {+ return;+ }+ try {+ const locationPath = await getLocationPath(currentLocation);+ const ftPath = getMetaFullTextFilePath(locationPath);+ const ftContent = await currentLocation.loadTextFilePromise(ftPath);+ if (ftContent) {+ // Parse JSONL format (new) or JSON object (old backward compat)+ const trimmed = ftContent.trim();+ let ftMap: Record<string, string>;+ if (trimmed.startsWith('{') && !trimmed.startsWith('{"p"')) {+ // Old JSON object format: {"relative/path": "text content", ...}+ try {+ ftMap = JSON.parse(trimmed);+ } catch (e) {+ ftMap = parseFullTextJsonl(ftContent);+ }+ } else {+ ftMap = parseFullTextJsonl(ftContent);+ }++ if (ftMap && typeof ftMap === 'object') {+ // Fulltext stores relative paths as keys, but index entries have+ // absolute paths (after enhanceDirectoryIndex). Convert to absolute.+ const sep = currentLocation.getDirSeparator();+ const absoluteFtMap: Record<string, string> = {};+ for (const [relPath, content] of Object.entries(ftMap)) {+ const absPath = joinPaths(sep, locationPath, relPath);+ absoluteFtMap[absPath] = content as string;+ }+ fullTextMap.current = absoluteFtMap;+ mergeFullTextIntoIndex(searchIndex, absoluteFtMap);+ enhancedIndex.current = undefined;+ fuseInstance.current = undefined;+ }+ }+ } catch (e: any) {+ // tsft.jsonl may not exist (no fulltext indexing enabled)+ console.log('No fulltext index found (tsft.jsonl)', e?.message || e);+ }+ fullTextLoaded.current = true;+ }++ async function getSearchResults(searchIndex: TS.FileSystemEntry[],searchQuery: TS.SearchQuery,): Promise<TS.FileSystemEntry[]> {- return Search.searchLocationIndex(searchIndex, searchQuery, tagDelimiter)+ // Lazy-load fulltext index only when text search is needed+ const hasTextQuery =+ searchQuery.textQuery && searchQuery.textQuery.length > 1;+ if (hasTextQuery && !fullTextLoaded.current) {+ await loadFullTextIfNeeded(searchIndex);+ }++ const prepared = getOrCreateEnhancedIndex(+ searchIndex,+ searchQuery.showUnixHiddenEntries,+ );+ // Build Fuse lazily only when text query is present+ const fuse = hasTextQuery ? getOrCreateFuse(prepared) : undefined;++ return Search.searchLocationIndex(searchIndex, searchQuery, tagDelimiter, {+ preparedIndex: prepared,+ fuseInstance: fuse,+ }).then((searchResults) => {return searchResults;}).catch((err) => {- // dispatch(AppActions.hideNotifications());console.log('Searching Index failed: ', err);showNotification(t('core:searchingFailed') + ' ' + err.message,@@ -716,63 +843,73 @@ export const LocationIndexContextProvider = ({showNotification(t('core:searching'), 'default', false, 'TIDSearching');walkingRef.current = true;- //let searchResultCount = 0;let searchResults = [];let maxSearchResultReached = false;- const searchingLocation = workSpace+ const searchingLocations = workSpace? locations.filter((l) => l.workSpaceId === workSpace.uuid): locations;- const result = searchingLocation.reduce(- (accumulatorPromise, location) =>- accumulatorPromise.then(async () => {- // cancel search if max search result count reached- if (searchResults.length >= searchQuery.maxSearchResults) {- maxSearchResultReached = true;- return Promise.resolve();- }- const nextPath = await getLocationPath(location);- const isCloudLocation = location.type === locationType.TYPE_CLOUD;- let directoryIndex = await loadIndexFromDisk(nextPath, location.uuid);- //console.log('Searching in: ' + nextPath);- showNotification(- t('core:searching') + ' ' + location.name,- 'default',- false,- 'TIDSearching',- );- if (- !location.disableIndexing &&- (!directoryIndex ||- directoryIndex.length < 1 ||- searchQuery.forceIndexing)- ) {- console.log('Creating index for : ' + nextPath);- directoryIndex = await createDirectoryIndexWrapper(- {- path: nextPath,- locationID: location.uuid,- ...(isCloudLocation && { bucketName: location.bucketName }),- },- location.fullTextIndex,- location.ignorePatternPaths,- enableWS,- );- }- return getSearchResults(directoryIndex, searchQuery).then(- (results) => {- if (results.length > 0) {- searchResults = [...searchResults, ...results];- appendSearchResults(results);- }- return true;- },- );- }),- Promise.resolve(),+ // Sequential on mobile (memory), parallel on desktop/web+ const CONCURRENCY = AppConfig.isNativeMobile ? 1 : 3;++ const searchSingleLocation = async (location: CommonLocation) => {+ if (searchResults.length >= searchQuery.maxSearchResults) {+ maxSearchResultReached = true;+ return;+ }+ const nextPath = await getLocationPath(location);+ const isCloudLocation = location.type === locationType.TYPE_CLOUD;+ let directoryIndex = await loadIndexFromDisk(nextPath, location.uuid);+ showNotification(+ t('core:searching') + ' ' + location.name,+ 'default',+ false,+ 'TIDSearching',+ );++ if (+ !location.disableIndexing &&+ (!directoryIndex ||+ directoryIndex.length < 1 ||+ searchQuery.forceIndexing)+ ) {+ console.log('Creating index for : ' + nextPath);+ directoryIndex = await createDirectoryIndexWrapper(+ {+ path: nextPath,+ locationID: location.uuid,+ ...(isCloudLocation && { bucketName: location.bucketName }),+ },+ location.fullTextIndex,+ location.ignorePatternPaths,+ enableWS,+ );+ }+ const results = await getSearchResults(directoryIndex, searchQuery);+ if (results.length > 0) {+ searchResults = [...searchResults, ...results];+ appendSearchResults(results);+ }+ };++ // Execute with concurrency limiting+ const tasks = searchingLocations.map(+ (loc) => () => searchSingleLocation(loc),);+ const batchPromises = [];+ for (let i = 0; i < tasks.length; i += CONCURRENCY) {+ const batch = tasks.slice(i, i + CONCURRENCY);+ batchPromises.push(batch);+ }- result+ batchPromises+ .reduce(+ (acc, batch) =>+ acc.then(() =>+ Promise.allSettled(batch.map((task) => task())).then(() => {}),+ ),+ Promise.resolve(),+ ).then(() => {enhanceSearchEntries(searchResults);console.timeEnd('globalSearch');@@ -813,16 +950,30 @@ export const LocationIndexContextProvider = ({const exist = await cLocation.checkDirExist(metaDirectory);try {if (!exist) {- await cLocation.createDirectoryPromise(metaDirectory); // todo platformFacade?+ await cLocation.createDirectoryPromise(metaDirectory);}- const folderIndexPath =- metaDirectory +- cLocation?.getDirSeparator() +- AppConfig.folderIndexFile; // getMetaIndexFilePath(directoryPath);- return cLocation+ const sep = cLocation?.getDirSeparator();+ const folderIndexPath = metaDirectory + sep + AppConfig.folderIndexFile;+ const folderFullTextPath =+ metaDirectory + sep + AppConfig.folderFullTextFile;++ // Split: strip textContent from main index, collect into fulltext map+ const fullTextEntries: Record<string, string> = {};+ let hasFullText = false;+ const strippedIndex = directoryIndex.map((entry: any) => {+ if (entry && entry.textContent) {+ fullTextEntries[entry.path] = entry.textContent;+ hasFullText = true;+ const { textContent, ...rest } = entry;+ return rest;+ }+ return entry;+ });++ const saveIndex = cLocation.saveTextFilePromise({ ...param, path: folderIndexPath },- JSON.stringify(directoryIndex), // relativeIndex),+ JSON.stringify(strippedIndex),true,).then(() => {@@ -834,6 +985,28 @@ export const LocationIndexContextProvider = ({.catch((err) => {console.log('Error saving the index for ' + folderIndexPath, err);});++ if (hasFullText) {+ const saveFullText = cLocation+ .saveTextFilePromise(+ { ...param, path: folderFullTextPath },+ serializeFullTextJsonl(fullTextEntries),+ true,+ )+ .then(() => {+ console.log('Fulltext index persisted to ' + folderFullTextPath);+ })+ .catch((err) => {+ console.log(+ 'Error saving fulltext index for ' + folderFullTextPath,+ err,+ );+ });+ return Promise.all([saveIndex, saveFullText]).then(+ ([indexResult]) => indexResult,+ );+ }+ return saveIndex;} catch (e) {console.log('Error saving the index', e);}