TagSpaces is an offline, open source, document manager with tagging support
fix: issue with creating sometimes absolute paths in the indexes
Ilian Sapundshievc57f79ec00b5
diff --git a/CLAUDE.md b/CLAUDE.mdindex 64f942dbd..7843587dc 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -121,6 +121,7 @@ tagspaces-common-node (Node.js fs implementation, injected as IO provider)- **Fulltext lazy loading** (`loadFullTextIfNeeded`): only runs when `searchQuery.textQuery` is present and the index isn't already loaded. Must convert `tsft.jsonl` keys from relative→absolute paths before calling `mergeFullTextIntoIndex`.- **Caches** (`enhancedIndex.current`, `fuseInstance.current`, `fullTextMap.current`) must all be invalidated together in `setIndex`. Missing any one causes stale search results.- **Progress feedback**: `createIndex` and `createIncrementalIndex` accept an `onProgress({count, entry})` callback. The renderer throttles updates to 250ms to avoid flooding React with re-renders.+- **Index path shape**: `index.current` holds the **enhanced** (absolute-path) form returned by `enhanceDirectoryIndex`. `tsi.json` / `tsft.jsonl` on disk hold **relative** paths. The renderer's `persistIndex` runs `cleanRootPath(entry.path, directoryPath, sep)` on every entry before writing, so it is safe to call with either shape. This matters because `reflectCreateEntry` / `reflectDeleteEntry` / `reflectUpdateEntry` / `reflectUpdateSidecarMeta` / `clearDirectoryIndex(true)` all pass `index.current` (absolute) straight into `persistIndex`. If you add a new code path that writes the index directly (bypassing `persistIndex`), you must strip the root prefix yourself — otherwise `enhanceDirectoryIndex` double-joins on the next load and search misses everything.### Local dev: syncing changes to node_modulesdiff --git a/src/renderer/hooks/LocationIndexContextProvider.tsx b/src/renderer/hooks/LocationIndexContextProvider.tsxindex 710cc6597..7af6a3582 100644--- a/src/renderer/hooks/LocationIndexContextProvider.tsx+++ b/src/renderer/hooks/LocationIndexContextProvider.tsx@@ -792,9 +792,13 @@ export const LocationIndexContextProvider = ({// Fulltext stores relative paths; index entries have absolute// paths after enhanceDirectoryIndex. Convert keys to absolute// so mergeFullTextIntoIndex matches by entry.path.+ // Defensive: legacy tsft.jsonl files (from before persist+ // normalization) may contain absolute keys — cleanRootPath+ // first so the subsequent join doesn't double the prefix.const sep = currentLocation.getDirSeparator();const absoluteFtMap: Record<string, string> = {};- for (const [relPath, content] of Object.entries(ftMap)) {+ for (const [storedKey, content] of Object.entries(ftMap)) {+ const relPath = cleanRootPath(storedKey, locationPath, sep);const absPath = joinPaths(sep, locationPath, relPath);absoluteFtMap[absPath] = content as string;}@@ -1046,17 +1050,26 @@ export const LocationIndexContextProvider = ({const folderFullTextPath =metaDirectory + sep + AppConfig.folderFullTextFile;+ // Persisted tsi.json / tsft.jsonl must store paths relative to the+ // location root. Callers may hand us either relative entries (fresh+ // output of createIndex / createIncrementalIndex) or the enhanced+ // absolute-path entries cached in index.current (reflectCreate/+ // reflectDelete/reflectUpdate/reflectUpdateSidecarMeta all persist+ // index.current). cleanRootPath is a no-op when the root isn't a+ // prefix, so running it unconditionally is safe for both inputs.// Split: strip textContent from main index, collect into fulltext mapconst fullTextEntries: Record<string, string> = {};let hasFullText = false;const strippedIndex = directoryIndex.map((entry: any) => {- if (entry && entry.textContent) {- fullTextEntries[entry.path] = entry.textContent;+ if (!entry) return entry;+ const relPath = cleanRootPath(entry.path, directoryPath, sep);+ if (entry.textContent) {+ fullTextEntries[relPath] = entry.textContent;hasFullText = true;const { textContent, ...rest } = entry;- return rest;+ return { ...rest, path: relPath };}- return entry;+ return entry.path === relPath ? entry : { ...entry, path: relPath };});const saveIndex = cLocation@@ -1107,17 +1120,23 @@ export const LocationIndexContextProvider = ({folderPath,): TS.FileSystemEntry[] {const loc = findLocation(locationID);- return directoryIndex.map((i: TS.FileSystemEntry) => ({- ...i,- locationID,- path: joinPaths(- loc.getDirSeparator(),- folderPath,- AppConfig.isWin- ? i.path.replaceAll('/', loc.getDirSeparator())- : i.path, //toPlatformPath()- ),- }));+ const sep = loc ? loc.getDirSeparator() : AppConfig.dirSeparator;+ return directoryIndex.map((i: TS.FileSystemEntry) => {+ // Defensive: some legacy tsi.json files (from before the persist+ // normalization was in place) contain absolute paths. Running+ // cleanRootPath first yields the relative form in both cases and+ // avoids a double-join like "/a/b/c" + "/a/b/c/file" → "/a/b/c/a/b/c/file".+ const relPath = cleanRootPath(i.path, folderPath, sep);+ return {+ ...i,+ locationID,+ path: joinPaths(+ sep,+ folderPath,+ AppConfig.isWin ? relPath.replaceAll('/', sep) : relPath,+ ),+ };+ });}function loadIndexFromDisk(diff --git a/tests/e2e/location.pw.e2e.js b/tests/e2e/location.pw.e2e.jsindex 881171437..8621c93bd 100644--- a/tests/e2e/location.pw.e2e.js+++ b/tests/e2e/location.pw.e2e.js@@ -1,4 +1,6 @@/* Copyright (c) 2016-present - TagSpaces GmbH. All rights reserved. */+import pathLib from 'path';+import fse from 'fs-extra';import AppConfig from '../../src/renderer/AppConfig';import { expect, test } from './fixtures';import {@@ -23,6 +25,7 @@ import {openLocationMenu,startupLocation} from './location.helpers';+import { getS3File } from '../s3rver/S3DataRefresh';import { clearDataStorage } from './welcome.helpers';export const perspectiveGridTable = '//*[@data-tid="perspectiveGridFileTable"]';@@ -210,6 +213,126 @@ test.describe('TST03 - Testing locations:', () => {8000,);});++ test('TST0329 - Index must persist only relative paths (worker + non-worker) [web,s3,electron]', async ({+ isS3,+ testDataDir,+ }) => {+ // Covers all three persist paths:+ // * electron-local → WS worker (indexer package persistIndex)+ // * electron-s3 → renderer persistIndex (non-worker, S3 IO)+ // * web-s3 → renderer persistIndex (non-worker, S3 IO)+ // The invariant we assert: tsi.json on disk must only contain paths+ // relative to the location root. Absolute paths would break search+ // on the next load because enhanceDirectoryIndex double-joins the+ // location path onto entries.+ test.setTimeout(180000);++ const readTsiJson = async () => {+ const rel = AppConfig.metaFolder + '/' + AppConfig.folderIndexFile;+ try {+ if (isS3) {+ return await getS3File(rel);+ }+ return await fse.readFile(+ pathLib.join(testDataDir, rel),+ 'utf-8',+ );+ } catch (e) {+ return undefined;+ }+ };++ // An earlier test in the same worker may have left a tsi.json behind+ // (testDataDir fixture is worker-scoped). Snapshot it so we can wait+ // for indexLocation to replace it before reading.+ const before = await readTsiJson();++ await openLocationMenu(testLocationName);+ await clickOn('[data-tid=indexLocation]');++ let after;+ await expect+ .poll(+ async () => {+ after = await readTsiJson();+ return !!after && after !== before;+ },+ { timeout: 120000, intervals: [500, 1000, 2000] },+ )+ .toBeTruthy();++ const entries = JSON.parse(after);+ expect(Array.isArray(entries)).toBe(true);+ expect(entries.length).toBeGreaterThan(0);++ // Absolute-path shapes we must never find in the persisted index:+ // Unix / S3-on-disk: leading "/"+ // Windows: drive letter like "C:\" or "C:/"+ const absPathRegex = /^(?:\/|[a-zA-Z]:[\\/])/;++ const offenders = entries.filter(+ (e) =>+ !e ||+ typeof e.path !== 'string' ||+ absPathRegex.test(e.path) ||+ // Belt-and-suspenders for unusual separator combinations: the+ // filesystem path of the location itself must not leak in.+ // Skipped for S3 — there the backing-store path and the bucket+ // root coincide by design.+ (!isS3 && e.path.includes(testDataDir)),+ );+ if (offenders.length > 0) {+ throw new Error(+ 'tsi.json contains entries with absolute or location-prefixed paths: ' ++ JSON.stringify(offenders.slice(0, 5), null, 2) ++ (offenders.length > 5+ ? `\n…and ${offenders.length - 5} more.`+ : ''),+ );+ }++ // Best-effort: the fulltext index (tsft.jsonl) shares the same+ // relative-path contract. Only written when full-text indexing is+ // enabled, so its absence is fine — but if it exists, its `p` keys+ // must pass the same check.+ const ftRel =+ AppConfig.metaFolder + '/' + AppConfig.folderFullTextFile;+ let ftContent;+ try {+ ftContent = isS3+ ? await getS3File(ftRel)+ : await fse.readFile(pathLib.join(testDataDir, ftRel), 'utf-8');+ } catch (e) {+ ftContent = undefined;+ }+ if (ftContent) {+ const ftLines = ftContent.split('\n').filter((l) => l.trim());+ const ftOffenders = [];+ for (const line of ftLines) {+ let parsed;+ try {+ parsed = JSON.parse(line);+ } catch (e) {+ continue;+ }+ if (+ parsed &&+ typeof parsed.p === 'string' &&+ (absPathRegex.test(parsed.p) ||+ (!isS3 && parsed.p.includes(testDataDir)))+ ) {+ ftOffenders.push(parsed.p);+ }+ }+ if (ftOffenders.length > 0) {+ throw new Error(+ 'tsft.jsonl contains absolute or location-prefixed keys: ' ++ JSON.stringify(ftOffenders.slice(0, 5)),+ );+ }+ }+ });});/*describe.skip('TST03 - Testing locations - test Context Menu in Tree Directory', () => {