Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): synchronize recent files after editor updates and deletions
johaven2b87607e871a
diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.tsindex 37958171..92b892e3 100644--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts@@ -9,8 +9,10 @@ import { ContextManager } from '../../../../infrastructure/context/services/contimport { SPACE_OPERATION } from '../../../spaces/constants/spaces'import type { SpaceEnv } from '../../../spaces/models/space-env.model'import type { UserModel } from '../../../users/models/user.model'+import { ACTION } from '../../../../common/constants'import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'import { FILE_MODE } from '../../constants/operations'+import { FileEvent } from '../../events/file-events'import { FileLockProps } from '../../interfaces/file-props.interface'import { LockConflict } from '../../models/file-lock-error'import { FilesLockManager } from '../../services/files-lock-manager.service'@@ -338,6 +340,7 @@ describe(CollaboraOnlineManager.name, () => {vi.spyOn(fs, 'stat').mockResolvedValue(mockStats as any)vi.spyOn(filesUtils, 'fileSize').mockResolvedValue(1024)+ const emitSpy = vi.spyOn(FileEvent, 'emit')const result = await service.saveDocument(mockRequest)@@ -347,6 +350,13 @@ describe(CollaboraOnlineManager.name, () => {expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/tmp/document-unique.docx', mockRequest.raw)expect(filesUtils.copyFileContent).toHaveBeenCalledWith('/tmp/document-unique.docx', mockSpace.realPath)expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')+ expect(emitSpy).toHaveBeenCalledWith('event', {+ user: mockUser,+ space: mockSpace,+ action: ACTION.UPDATE,+ rPath: mockSpace.realPath,+ source: 'editor'+ })})it('should throw error when document size mismatch', async () => {diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.tsindex 1b8b72a3..ae35d09c 100644--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts@@ -136,7 +136,7 @@ export class CollaboraOnlineManager {try {await copyFileContent(tmpFilePath, req.space.realPath)// emit file event- FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath })+ FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath, source: 'editor' })await removeFiles(tmpFilePath)const fStats = await fs.stat(req.space.realPath)return { LastModifiedTime: fStats.mtime.toISOString() } satisfies CollaboraSaveDocumentDtodiff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.tsindex b01b951e..c4101010 100644--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts@@ -11,8 +11,10 @@ import { Cache } from '../../../../infrastructure/cache/cache.service'import { ContextManager } from '../../../../infrastructure/context/services/context-manager.service'import type { SpaceEnv } from '../../../spaces/models/space-env.model'import type { UserModel } from '../../../users/models/user.model'+import { ACTION } from '../../../../common/constants'import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'import { FILE_MODE } from '../../constants/operations'+import { FileEvent } from '../../events/file-events'import { LockConflict } from '../../models/file-lock-error'import { FilesLockManager } from '../../services/files-lock-manager.service'import * as filesUtils from '../../utils/files'@@ -275,6 +277,7 @@ describe(OnlyOfficeManager.name, () => {}const expectSuccessfulSaveCallback = async (url: string, callbackData: Record<string, any> = {}) => {+ const emitSpy = vi.spyOn(FileEvent, 'emit')jwtService.verifyAsync.mockResolvedValue({status: 2,actions: [],@@ -288,6 +291,13 @@ describe(OnlyOfficeManager.name, () => {const result = await service.callBack(mockUser, mockSpaceEnv, mockToken)expect(result).toEqual({ error: 0 })+ expect(emitSpy).toHaveBeenCalledWith('event', {+ user: mockUser,+ space: mockSpaceEnv,+ action: ACTION.UPDATE,+ rPath: mockSpaceEnv.realPath,+ source: 'editor'+ })expect(httpService.axiosRef).toHaveBeenCalledWith(expect.objectContaining({ url, maxRedirects: 0 }))}diff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.tsindex fd21d814..1e11b8c1 100644--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.ts@@ -384,7 +384,7 @@ export class OnlyOfficeManager {try {await copyFileContent(tmpFilePath, space.realPath)// emit file event- FileEvent.emit('event', { user: user, space: space, action: ACTION.UPDATE, rPath: space.realPath })+ FileEvent.emit('event', { user: user, space: space, action: ACTION.UPDATE, rPath: space.realPath, source: 'editor' })await removeFiles(tmpFilePath)} catch (e) {throw new Error(`unable to save document : ${e.message}`)diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.tsindex 3061c6a4..60d448fb 100644--- a/backend/src/applications/files/interfaces/file-event.interface.ts+++ b/backend/src/applications/files/interfaces/file-event.interface.ts@@ -11,6 +11,7 @@ export interface FileEventType {space: SpaceEnvaction: ACTIONrPath: string+ source?: 'editor'}export interface FileEventEmit {diff --git a/backend/src/applications/files/interfaces/file-recent-location.interface.ts b/backend/src/applications/files/interfaces/file-recent-location.interface.tsdeleted file mode 100644index 2d4acc9e..00000000--- a/backend/src/applications/files/interfaces/file-recent-location.interface.ts+++ /dev/null@@ -1,6 +0,0 @@-export interface FileRecentLocation {- ownerId?: number- spaceId?: number- shareId?: number | number[]- path: string-}diff --git a/backend/src/applications/files/schemas/file-recent.interface.ts b/backend/src/applications/files/schemas/file-recent.interface.tsindex f165716f..984647f9 100644--- a/backend/src/applications/files/schemas/file-recent.interface.ts+++ b/backend/src/applications/files/schemas/file-recent.interface.ts@@ -3,6 +3,7 @@ import { filesRecents } from './files-recents.schema'type FileRecentSchema = typeof filesRecents.$inferSelectexport class FileRecent implements FileRecentSchema {+ // Browser file identity: database id when available, otherwise negative inode.id: numberownerId: numberspaceId: number@@ -12,3 +13,20 @@ export class FileRecent implements FileRecentSchema {mime: stringmtime: number}++export interface FileRecentLocation {+ ownerId?: number+ spaceId?: number+ shareId?: number | number[]+ path: string+}++export type FileRecentUpdate = Pick<FileRecent, 'id'> & Partial<Pick<FileRecent, 'name' | 'mime' | 'mtime'>>++export interface FileRecentDeletion {+ userId: number+ spaceId: number+ inPersonalSpace: boolean+ inSharesRepository: boolean+ path: string+}diff --git a/backend/src/applications/files/schemas/files.schema.ts b/backend/src/applications/files/schemas/files.schema.tsindex 2b930a9f..19632db7 100644--- a/backend/src/applications/files/schemas/files.schema.ts+++ b/backend/src/applications/files/schemas/files.schema.ts@@ -1,4 +1,4 @@-import { SQL, sql } from 'drizzle-orm'+import { SQL, sql, SQLWrapper } from 'drizzle-orm'import { AnyMySqlColumn, bigint, boolean, index, mysqlTable, varchar } from 'drizzle-orm/mysql-core'import { escapeSQLRegexp } from '../../../common/functions'import { shares } from '../../shares/schemas/shares.schema'@@ -46,7 +46,10 @@ export const files = mysqlTable(// supports the case where path = '.' or './sync-in' and removes '.' or './' if existsexport const filePathSQL = (file: any): SQL<string> => sql`REGEXP_REPLACE(CONCAT(${file.path}, '/', ${file.name}), '^(\\\\.\\\\/){0,1}(.*)', '\\\\2')`-export const childFilesFindRegexp = (path: string): SQL<string> => sql`${files.path}${sql.raw(`REGEXP '^${escapeSQLRegexp(path)}(/|$)'`)}`+export const childPathFindRegexp = (pathSQL: SQLWrapper, path: string): SQL<string> =>+ sql`${pathSQL}${sql.raw(`REGEXP '^${escapeSQLRegexp(path)}(/|$)'`)}`++export const childFilesFindRegexp = (path: string): SQL<string> => childPathFindRegexp(files.path, path)export const childFilesReplaceRegexp = (srcPath: string, dstPath: string): SQL<string> =>sql`REGEXP_REPLACE(${files.path}, '^${sql.raw(escapeSQLRegexp(srcPath))}', '${sql.raw(escapeSQLRegexp(dstPath))}')`diff --git a/backend/src/applications/files/services/files-event-manager.service.spec.ts b/backend/src/applications/files/services/files-event-manager.service.spec.tsindex ef11911d..b6c270fe 100644--- a/backend/src/applications/files/services/files-event-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-event-manager.service.spec.ts@@ -10,16 +10,27 @@ import { quotaCacheKeyFromSpace } from '../utils/quota'import { indexingUpdateCacheKeysFromSpace } from '../utils/indexing'import { configuration } from '../../../configuration/config.environment'import type { Mock } from 'vitest'+import { FilesRecents } from './files-recents.service'describe(FilesEventManager.name, () => {let service: FilesEventManagerlet cacheSetMock: Mock<(key: string, value: unknown, ttl?: number) => Promise<boolean>>+ let deleteRecentsMock: Mock+ let updateRecentFromEditorMock: Mocklet contentIndexingEnabled: booleanconst buildEvent = (props?: Partial<FileEventType>): FileEventType =>({user: { id: 7, login: 'john' } as any,- space: { id: 13, alias: 'project', inPersonalSpace: false, inSharesRepository: false, root: {} } as any,+ space: {+ id: 13,+ alias: 'project',+ url: 'files/project/document.bin',+ inTrashRepository: false,+ inPersonalSpace: false,+ inSharesRepository: false,+ root: {}+ } as any,action: ACTION.ADD,rPath: '/files/document.bin',...props@@ -27,12 +38,18 @@ describe(FilesEventManager.name, () => {beforeEach(async () => {cacheSetMock = vi.fn().mockResolvedValue(true)+ deleteRecentsMock = vi.fn().mockResolvedValue(undefined)+ updateRecentFromEditorMock = vi.fn().mockResolvedValue(undefined)const module: TestingModule = await Test.createTestingModule({providers: [FilesEventManager,{provide: Cache,useValue: { set: cacheSetMock }+ },+ {+ provide: FilesRecents,+ useValue: { deleteRecents: deleteRecentsMock, updateRecentFromEditor: updateRecentFromEditorMock }}]}).compile()@@ -44,13 +61,8 @@ describe(FilesEventManager.name, () => {afterEach(async () => {configuration.applications.files.contentIndexing.enabled = contentIndexingEnabled- vi.useRealTimers()- vi.clearAllMocks()await service.onModuleDestroy()- })-- it('should be defined', () => {- expect(service).toBeDefined()+ vi.restoreAllMocks()})it('should cache a quota key once for duplicate non-delete events', async () => {@@ -62,17 +74,60 @@ describe(FilesEventManager.name, () => {expect(cacheSetMock).toHaveBeenCalledWith(quotaCacheKeyFromSpace(event.user.id, event.space, true), true, CACHE_QUOTA_TTL)})+ it('should delete recents before processing only the latest editor update per path', async () => {+ const callOrder: string[] = []+ deleteRecentsMock.mockImplementationOnce(async () => {+ callOrder.push('delete')+ })+ updateRecentFromEditorMock.mockImplementationOnce(async () => {+ callOrder.push('update')+ })+ const first = buildEvent({ action: ACTION.UPDATE, source: 'editor', rPath: '/files/first.bin' })+ const latest = { ...first, rPath: '/files/latest.bin' }++ await (service as any).processEvents([first, buildEvent({ action: ACTION.UPDATE }), buildEvent({ action: ACTION.DELETE }), latest])++ expect(deleteRecentsMock).toHaveBeenCalledWith([+ { userId: 7, spaceId: 13, inPersonalSpace: false, inSharesRepository: false, path: 'files/project/document.bin' }+ ])+ expect(updateRecentFromEditorMock).toHaveBeenCalledOnce()+ expect(updateRecentFromEditorMock).toHaveBeenCalledWith(latest.user, latest.space, latest.rPath)+ expect(callOrder).toEqual(['delete', 'update'])+ })++ it('should ignore recent deletion errors', async () => {+ deleteRecentsMock.mockRejectedValueOnce(new Error('database unavailable'))++ await expect((service as any).processEvents([buildEvent({ action: ACTION.DELETE })])).resolves.toBeUndefined()+ })+it('should ignore quota cache update for delete events', async () => {await (service as any).processEvents([buildEvent({ action: ACTION.DELETE, rPath: '/files/deleted.tmp' })])expect(cacheSetMock).not.toHaveBeenCalled()})+ it('should ignore deletes from trash and shares list for recents', async () => {+ const event = buildEvent()+ await (service as any).processEvents([+ buildEvent({ action: ACTION.DELETE, space: { ...event.space, inTrashRepository: true } as any }),+ buildEvent({ action: ACTION.DELETE, space: { ...event.space, inSharesList: true } as any })+ ])++ expect(deleteRecentsMock).not.toHaveBeenCalled()+ })+it('should cache quota and indexing keys for indexable files', async () => {configuration.applications.files.contentIndexing.enabled = trueconst event = buildEvent({rPath: '/shares/spec.pdf',- space: { id: 42, alias: 'shared-space', inPersonalSpace: false, inSharesRepository: true, root: {} } as any+ space: {+ id: 42,+ alias: 'shared-space',+ inPersonalSpace: false,+ inSharesRepository: true,+ root: {}+ } as any})const quotaKey = quotaCacheKeyFromSpace(event.user.id, event.space, true)const indexingKeys = indexingUpdateCacheKeysFromSpace(event.user.id, event.space)diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.tsindex 8520b835..33abe4c9 100644--- a/backend/src/applications/files/services/files-event-manager.service.ts+++ b/backend/src/applications/files/services/files-event-manager.service.ts@@ -9,10 +9,12 @@ import { getExtensionWithoutDot } from '../utils/files'import { CACHE_INDEXING_EVENT_TTL, INDEXABLE_EXTENSIONS } from '../constants/indexing'import { indexingUpdateCacheKeysFromSpace } from '../utils/indexing'import { configuration } from '../../../configuration/config.environment'+import { FilesRecents } from './files-recents.service'@Injectable()export class FilesEventManager implements OnModuleDestroy {/* Used to:+ - reconcile recents after deletions and editor saves- store cached events for storage usage updates- store indexing events for full-text index updates- todo: handle versioning@@ -27,7 +29,10 @@ export class FilesEventManager implements OnModuleDestroy {private isFlushing = falseprivate flushRequested = false- constructor(private readonly cache: Cache) {+ constructor(+ private readonly cache: Cache,+ private readonly filesRecents: FilesRecents+ ) {FileEvent.on('event', this.onFileEvent)}@@ -50,6 +55,15 @@ export class FilesEventManager implements OnModuleDestroy {this.startFlushTimer()}+ private async processRecentEditorUpdate(event: FileEventType): Promise<void> {+ try {+ await this.filesRecents.updateRecentFromEditor(event.user, event.space, event.rPath)+ } catch (e) {+ if ((e as NodeJS.ErrnoException)?.code === 'ENOENT') return+ this.logger.warn({ tag: this.processRecentEditorUpdate.name, msg: `Could not update recent file ${event.space.url}: ${e}` })+ }+ }+private startFlushTimer(): void {if (this.flushTimer) {return@@ -106,7 +120,45 @@ export class FilesEventManager implements OnModuleDestroy {this.logger.warn({ tag: this.processEvents.name, msg: `Could not process event: ${JSON.stringify(event)} - ${e}` })}}- await this.storeEventsInCache()+ await Promise.all([this.storeEventsInCache(), this.processRecentEvents(fEvents)])+ }++ private async processRecentEvents(events: FileEventType[]): Promise<void> {+ // Delete first, then let editor updates reinsert only files that still exist on disk.+ await this.processRecentDeletes(events)+ await this.processRecentEditorUpdates(events)+ }++ private async processRecentEditorUpdates(events: FileEventType[]): Promise<void> {+ const updatesByPath = new Map<string, FileEventType>()+ for (const event of events) {+ // Only editor saves update recents from events; other updates are reconciled while browsing.+ if (event.action !== ACTION.UPDATE || event.source !== 'editor') continue+ const key = `${event.user.id}:${event.space.id}:${event.space.inPersonalSpace}:${event.space.inSharesRepository}:${event.space.url}`+ // Keep the latest save per logical path to perform at most one stat and upsert per buffered batch.+ updatesByPath.set(key, event)+ }+ await Promise.all([...updatesByPath.values()].map((event) => this.processRecentEditorUpdate(event)))+ }++ private async processRecentDeletes(events: FileEventType[]): Promise<void> {+ const deletions = events+ .filter(+ ({ action, space }) => (action === ACTION.DELETE || action === ACTION.DELETE_PERMANENTLY) && !space.inTrashRepository && !space.inSharesList+ )+ .map(({ user, space }) => ({+ userId: user.id,+ spaceId: space.id,+ inPersonalSpace: space.inPersonalSpace,+ inSharesRepository: space.inSharesRepository,+ path: space.url+ }))+ if (!deletions.length) return+ try {+ await this.filesRecents.deleteRecents(deletions)+ } catch (e) {+ this.logger.warn({ tag: this.processRecentDeletes.name, msg: `Could not delete recent files: ${e}` })+ }}private async storeEventsInCache() {diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex 50dc68d1..0c360f9f 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -365,7 +365,10 @@ describe(FilesManager.name, () => {expect(tmpWritePath.endsWith('-report.txt')).toBe(true)expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, expect.anything())expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, '/data/users/john/files/report.txt', true)- expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.UPDATE, rPath: '/data/users/john/files/report.txt' }))+ expect(emitSpy).toHaveBeenCalledWith(+ 'event',+ expect.objectContaining({ action: ACTION.UPDATE, rPath: '/data/users/john/files/report.txt', source: 'editor' })+ )})it('should reject PATCH when destination does not exist', async () => {@@ -416,6 +419,7 @@ describe(FilesManager.name, () => {expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, space.realPath, true)expect(filesUtils.removeFiles).not.toHaveBeenCalled()expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.UPDATE, rPath: space.realPath }))+ expect(emitSpy).not.toHaveBeenCalledWith('event', expect.objectContaining({ source: 'editor' }))})it('should create missing destination directory and release created lock for POST nested upload', async () => {@@ -1013,11 +1017,13 @@ describe(FilesManager.name, () => {})const trashFile = '/data/users/john/trash/documents/document.txt'prepareFileTransfer(space.realPath, trashFile)+ const emitSpy = vi.spyOn(FileEvent, 'emit')await service.delete(user, space)expect(filesUtils.moveFiles).toHaveBeenCalledWith(space.realPath, trashFile, true)expect(filesTasksTransfer.delete).not.toHaveBeenCalled()+ expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.DELETE, rPath: trashFile })})})diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex a7543766..0f00fd50 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -306,7 +306,13 @@ export class FilesManager {if (fileWritten) {// Emit only after the final destination has been written or moved into place.const fileEventAction: ACTION = patchMethod || (dstExists && !dstIsDir) ? ACTION.UPDATE : ACTION.ADD- FileEvent.emit('event', { user, space, action: fileEventAction, rPath: dstFile })+ FileEvent.emit('event', {+ user,+ space,+ action: fileEventAction,+ rPath: dstFile,+ ...(patchMethod && { source: 'editor' as const })+ })}if (!patchMethod && created) {// Remove the file lock only if it has not been refreshed@@ -557,6 +563,10 @@ export class FilesManager {const userSpace = new SpaceEnv(SPACE_PERSONAL, null, false)userSpace.setup(user, SPACE_REPOSITORY.TRASH, null, [], [])FileEvent.emit('event', { user, space: userSpace, action: ACTION.ADD, rPath: trashFile })+ } else {+ // emit an event for the file or directory moved to the trash+ // space keeps its original path and rPath is its new trash path+ FileEvent.emit('event', { user, space, action: ACTION.DELETE, rPath: trashFile })}} else {// unsupported case: delete the file (this shouldn't happen)diff --git a/backend/src/applications/files/services/files-queries.service.ts b/backend/src/applications/files/services/files-queries.service.tsindex 66525039..9cb34502 100644--- a/backend/src/applications/files/services/files-queries.service.ts+++ b/backend/src/applications/files/services/files-queries.service.ts@@ -1,6 +1,5 @@import { Inject, Injectable, Logger } from '@nestjs/common'import { and, desc, eq, getTableColumns, inArray, isNull, or, SelectedFields, sql, SQL } from 'drizzle-orm'-import { popFromObject } from '../../../common/shared'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'import { concatDistinctObjectsInArray, convertToWhere, dbCheckAffectedRows, dbGetInsertedId } from '../../../infrastructure/database/utils'@@ -12,12 +11,11 @@ import { syncClients } from '../../sync/schemas/sync-clients.schema'import { syncPaths } from '../../sync/schemas/sync-paths.schema'import { FileDBProps } from '../interfaces/file-db-props.interface'import { FileProps } from '../interfaces/file-props.interface'-import { FileRecentLocation } from '../interfaces/file-recent-location.interface'-import { FileRecent } from '../schemas/file-recent.interface'+import type { FileRecent, FileRecentLocation, FileRecentUpdate } from '../schemas/file-recent.interface'import { File } from '../schemas/file.interface'import { filesRecents } from '../schemas/files-recents.schema'-import { childFilesFindRegexp, childFilesReplaceRegexp, filePathSQL, files } from '../schemas/files.schema'-import { dirName, fileName } from '../utils/files'+import { childFilesFindRegexp, childFilesReplaceRegexp, childPathFindRegexp, filePathSQL, files } from '../schemas/files.schema'+import { dirName, fileName, isPathInside } from '../utils/files'@Injectable()export class FilesQueries {@@ -308,7 +306,7 @@ export class FilesQueries {.limit(limit)}- getRecentsFromLocation(location: Partial<Record<keyof FileRecent, any>>): Promise<FileRecent[]> {+ getRecentsFromLocation(location: FileRecentLocation): Promise<FileRecent[]> {const where: SQL[] = convertToWhere(filesRecents, location)return this.db.select(getTableColumns(filesRecents))@@ -316,31 +314,73 @@ export class FilesQueries {.where(and(...where))}- async updateRecents(- location: FileRecentLocation,- add: Partial<FileRecent>[] | FileRecent[],- update: Record<string | 'object', Partial<FileProps> | FileProps>[],- remove: FileRecent['id'][]- ): Promise<void> {+ async deleteRecents(locations: FileRecentLocation[]): Promise<void> {+ const groups = new Map<string, { repository: Omit<FileRecentLocation, 'path'>; paths: Set<string> }>()+ for (const { path: sourcePath, ...repository } of locations) {+ const key = JSON.stringify(repository)+ const group = groups.get(key)+ if (group) {+ group.paths.add(sourcePath)+ } else {+ groups.set(key, { repository, paths: new Set([sourcePath]) })+ }+ }++ const batchSize = 50+ for (const { repository, paths } of groups.values()) {+ const compactPaths: string[] = []+ for (const sourcePath of [...paths].sort((a, b) => a.length - b.length)) {+ if (!compactPaths.some((parentPath) => isPathInside(parentPath, sourcePath, true))) compactPaths.push(sourcePath)+ }+ const where: SQL[] = convertToWhere(filesRecents, repository)+ for (let offset = 0; offset < compactPaths.length; offset += batchSize) {+ const batch = compactPaths.slice(offset, offset + batchSize)+ try {+ await this.db+ .delete(filesRecents)+ .where(and(...where, or(...batch.map((sourcePath) => childPathFindRegexp(filePathSQL(filesRecents), sourcePath)))))+ } catch (e) {+ this.logger.error({ tag: this.deleteRecents.name, msg: `${e}` })+ }+ }+ }+ }++ async replaceRecents(location: FileRecentLocation, recents: FileRecent[]): Promise<void> {+ const where: SQL[] = convertToWhere(filesRecents, location)+ await this.db.transaction(async (tx) => {+ await tx.delete(filesRecents).where(and(...where))+ if (recents.length) await tx.insert(filesRecents).values(recents)+ })+ }++ async upsertRecent(location: FileRecentLocation, recent: FileRecent): Promise<void> {+ const where: SQL[] = convertToWhere(filesRecents, location)+ await this.db.transaction(async (tx) => {+ // The table has no unique key: replacing by logical location also removes a stale inode identity.+ await tx.delete(filesRecents).where(and(...where, eq(filesRecents.name, recent.name)))+ await tx.insert(filesRecents).values(recent)+ })+ }++ async updateRecents(location: FileRecentLocation, add: FileRecent[], update: FileRecentUpdate[], remove: FileRecent['id'][]): Promise<void> {const where: SQL[] = convertToWhere(filesRecents, location)// addif (add.length) {try {- await this.db.insert(filesRecents).values(add as FileRecent[])+ await this.db.insert(filesRecents).values(add)} catch (e) {this.logger.error({ tag: this.updateRecents.name, msg: `${e}` })}}// updateif (update.length) {- for (const props of update) {- const f: FileProps = popFromObject('object', props)+ for (const { id, ...props } of update) {try {await this.db.update(filesRecents)- .set({ ...props })- .where(and(...where, eq(filesRecents.id, f.id)))- .limit(1)+ .set(props)+ .where(and(...where, eq(filesRecents.id, id)))} catch (e) {this.logger.error({ tag: this.updateRecents.name, msg: `${e}` })}diff --git a/backend/src/applications/files/services/files-recents.service.spec.ts b/backend/src/applications/files/services/files-recents.service.spec.tsindex b71289b3..d94b30be 100644--- a/backend/src/applications/files/services/files-recents.service.spec.ts+++ b/backend/src/applications/files/services/files-recents.service.spec.ts@@ -6,14 +6,19 @@ import { SpacesQueries } from '../../spaces/services/spaces-queries.service'import { USER_PERMISSION } from '../../users/constants/user'import { FilesQueries } from './files-queries.service'import { FilesRecents } from './files-recents.service'-import { Mock } from 'vitest'+import type { Mock } from 'vitest'+import * as filesUtils from '../utils/files'describe(FilesRecents.name, () => {let service: FilesRecentslet filesQueries: {getRecentsFromUser: MockgetRecentsFromLocation: Mock+ deleteRecents: Mock+ replaceRecents: MockupdateRecents: Mock+ getSpaceFileId: Mock+ upsertRecent: Mock}let spacesQueries: {spaceIds: Mock@@ -26,7 +31,11 @@ describe(FilesRecents.name, () => {filesQueries = {getRecentsFromUser: vi.fn().mockResolvedValue([]),getRecentsFromLocation: vi.fn().mockResolvedValue([]),- updateRecents: vi.fn().mockResolvedValue(undefined)+ deleteRecents: vi.fn().mockResolvedValue(undefined),+ replaceRecents: vi.fn().mockResolvedValue(undefined),+ updateRecents: vi.fn().mockResolvedValue(undefined),+ getSpaceFileId: vi.fn().mockResolvedValue(undefined),+ upsertRecent: vi.fn().mockResolvedValue(undefined)}spacesQueries = {spaceIds: vi.fn().mockResolvedValue([])@@ -51,7 +60,7 @@ describe(FilesRecents.name, () => {})afterEach(() => {- vi.clearAllMocks()+ vi.restoreAllMocks()})const userWithPermissions = (applications: USER_PERMISSION[] = [], props: Record<string, any> = {}) =>@@ -63,10 +72,6 @@ describe(FilesRecents.name, () => {...props}) as any- it('should be defined', () => {- expect(service).toBeDefined()- })-it('should load recents from user accessible spaces and shares', async () => {const recents = [{ id: 1, name: 'a.txt' }]spacesQueries.spaceIds.mockResolvedValueOnce([10, 11])@@ -127,6 +132,85 @@ describe(FilesRecents.name, () => {expect(filesQueries.updateRecents).not.toHaveBeenCalled()})+ it('should normalize and map recent deletions to their repositories', async () => {+ await service.deleteRecents([+ { userId: 5, spaceId: 0, inPersonalSpace: true, inSharesRepository: false, path: 'files/personal/docs/../old.txt' },+ { userId: 5, spaceId: 9, inPersonalSpace: false, inSharesRepository: false, path: 'files/project/archive' },+ { userId: 5, spaceId: 12, inPersonalSpace: false, inSharesRepository: true, path: 'shares/project/report.pdf' }+ ])++ expect(filesQueries.deleteRecents).toHaveBeenCalledWith([+ { ownerId: 5, path: 'files/personal/old.txt' },+ { spaceId: 9, path: 'files/project/archive' },+ { shareId: 12, path: 'shares/project/report.pdf' }+ ])+ })++ it('should stat and upsert one recent file after an editor update', async () => {+ const now = currentTimeStamp(null, true)+ const file = {+ id: -123,+ path: 'docs',+ name: 'report.txt',+ isDir: false,+ size: 42,+ ctime: now,+ mtime: now,+ mime: 'text/plain'+ }+ const user = { id: 5 } as any+ const space = {+ id: 9,+ url: 'files/project/docs/report.txt',+ dbFile: { spaceId: 9, path: 'docs/report.txt', inTrash: false },+ inTrashRepository: false,+ inSharesList: false,+ inPersonalSpace: false,+ inSharesRepository: false+ } as any+ vi.spyOn(filesUtils, 'getProps').mockResolvedValueOnce(file as any)+ filesQueries.getSpaceFileId.mockResolvedValueOnce(321)++ await service.updateRecentFromEditor(user, space, '/data/project/docs/report.txt')++ expect(filesUtils.getProps).toHaveBeenCalledWith('/data/project/docs/report.txt', 'docs/report.txt')+ expect(filesQueries.getSpaceFileId).toHaveBeenCalledWith(file, space.dbFile, { withDir: false })+ expect(filesQueries.upsertRecent).toHaveBeenCalledWith(+ { spaceId: 9, path: 'files/project/docs' },+ { id: 321, spaceId: 9, path: 'files/project/docs', name: 'report.txt', mtime: now, mime: 'text/plain' }+ )+ })++ it('should ignore an old file after an editor update', async () => {+ vi.spyOn(filesUtils, 'getProps').mockResolvedValueOnce({+ id: -123,+ path: 'docs',+ name: 'old.txt',+ isDir: false,+ size: 42,+ ctime: 0,+ mtime: currentTimeStamp(null, true) - convertHumanTimeToMs('30d'),+ mime: 'text/plain'+ } as any)++ await service.updateRecentFromEditor(+ { id: 5 } as any,+ {+ id: 0,+ url: 'files/personal/docs/old.txt',+ dbFile: { ownerId: 5, path: 'docs/old.txt', inTrash: false },+ inTrashRepository: false,+ inSharesList: false,+ inPersonalSpace: true,+ inSharesRepository: false+ } as any,+ '/data/users/john/files/docs/old.txt'+ )++ expect(filesQueries.getSpaceFileId).not.toHaveBeenCalled()+ expect(filesQueries.upsertRecent).not.toHaveBeenCalled()+ })+it('should skip persistence when filtered fs recents and db recents are both empty', async () => {const oldMtime = currentTimeStamp(null, true) - convertHumanTimeToMs('30d')const files = [@@ -145,12 +229,12 @@ describe(FilesRecents.name, () => {const now = currentTimeStamp(null, true)const location = { ownerId: 5, path: 'files/personal/docs' }const dbRecents = [- { id: 1, name: 'old-name.txt', mtime: now - 1000, mime: 'text-plain' },- { id: 2, name: 'remove-me.txt', mtime: now - 1000, mime: 'text-plain' }+ { id: 101, name: 'old-name.txt', mtime: now - 1000, mime: 'text-plain', ...location },+ { id: 102, name: 'remove-me.txt', mtime: now - 1000, mime: 'text-plain', ...location }]const fsFiles = [- { id: 1, name: 'new-name.txt', isDir: false, size: 100, mtime: now, mime: 'text-plain' },- { id: 3, name: 'add-me.txt', isDir: false, size: 200, mtime: now, mime: 'text-plain' }+ { id: 101, name: 'new-name.txt', isDir: false, size: 100, mtime: now, mime: 'application-pdf' },+ { id: 103, name: 'add-me.txt', isDir: false, size: 200, mtime: now, mime: 'text-plain' }]filesQueries.getRecentsFromLocation.mockResolvedValueOnce(dbRecents)@@ -158,38 +242,31 @@ describe(FilesRecents.name, () => {expect(filesQueries.getRecentsFromLocation).toHaveBeenCalledWith(location)expect(filesQueries.updateRecents).toHaveBeenCalledTimes(1)- const [loc, toAdd, toUpdate, toRemove] = filesQueries.updateRecents.mock.calls[0]- expect(loc).toEqual(location)- expect(toAdd).toEqual([{ id: 3, name: 'add-me.txt', mtime: now, mime: 'text-plain', ...location }])- expect(toUpdate).toEqual([{ name: 'new-name.txt', mtime: now, object: fsFiles[0] }])- expect(toRemove).toEqual([2])+ const [actualLocation, toAdd, toUpdate, toRemove] = filesQueries.updateRecents.mock.calls[0]+ expect(actualLocation).toEqual(location)+ expect(toAdd).toEqual([{ id: 103, name: 'add-me.txt', mtime: now, mime: 'text-plain', ...location }])+ expect(toUpdate).toEqual([{ id: 101, name: 'new-name.txt', mtime: now, mime: 'application-pdf' }])+ expect(toRemove).toEqual([102])})- it('should use share location list and per-file share id when browsing shares list', async () => {+ it('should replace the shares snapshot and deduplicate by file id', async () => {const now = currentTimeStamp(null, true)+ const space = { inTrashRepository: false, inSharesList: true, inSharesRepository: false, inPersonalSpace: false, id: 0, url: 'shares' } as anyconst files = [- { id: 10, name: 'a.txt', isDir: false, size: 5, mtime: now, mime: 'text-plain', root: { id: 101 } },- { id: 11, name: 'b.txt', isDir: false, size: 5, mtime: now, mime: 'text-plain', root: { id: 102 } }+ { id: 10, name: 'report.pdf', isDir: false, size: 5, mtime: now, mime: 'application-pdf', root: { id: 101 } },+ { id: 10, name: 'report.pdf', isDir: false, size: 5, mtime: now, mime: 'application-pdf', root: { id: 102 } }]- filesQueries.getRecentsFromLocation.mockResolvedValueOnce([])+ await service.updateRecents({ id: 5 } as any, space, files as any)- await service.updateRecents(- { id: 5 } as any,- { inTrashRepository: false, inSharesList: true, inSharesRepository: false, inPersonalSpace: false, id: 0, url: 'shares' } as any,- files as any- )-- const [location, toAdd] = filesQueries.updateRecents.mock.calls[0]- expect(location).toEqual({ shareId: [101, 102], path: 'shares' })- expect(toAdd).toEqual([- { id: 10, name: 'a.txt', mtime: now, mime: 'text-plain', path: 'shares', shareId: 101 },- { id: 11, name: 'b.txt', mtime: now, mime: 'text-plain', path: 'shares', shareId: 102 }+ expect(filesQueries.getRecentsFromLocation).not.toHaveBeenCalled()+ expect(filesQueries.updateRecents).not.toHaveBeenCalled()+ expect(filesQueries.replaceRecents).toHaveBeenCalledWith({ shareId: [101, 102], path: 'shares' }, [+ { id: 10, name: 'report.pdf', mtime: now, mime: 'application-pdf', path: 'shares', shareId: 102 }])})it('should use repository share id for a single share location', async () => {const now = currentTimeStamp(null, true)- filesQueries.getRecentsFromLocation.mockResolvedValueOnce([])await service.updateRecents({ id: 5 } as any,@@ -202,7 +279,6 @@ describe(FilesRecents.name, () => {it('should use space id location for regular space repository', async () => {const now = currentTimeStamp(null, true)- filesQueries.getRecentsFromLocation.mockResolvedValueOnce([])await service.updateRecents({ id: 5 } as any,diff --git a/backend/src/applications/files/services/files-recents.service.ts b/backend/src/applications/files/services/files-recents.service.tsindex eaf6a8eb..534696e3 100644--- a/backend/src/applications/files/services/files-recents.service.ts+++ b/backend/src/applications/files/services/files-recents.service.ts@@ -1,5 +1,6 @@import { Injectable } from '@nestjs/common'-import { convertDiffUpdate, convertHumanTimeToMs, diffCollection } from '../../../common/functions'+import path from 'node:path'+import { convertHumanTimeToMs } from '../../../common/functions'import { currentTimeStamp } from '../../../common/shared'import { SharesQueries } from '../../shares/services/shares-queries.service'import { SpaceEnv } from '../../spaces/models/space-env.model'@@ -7,13 +8,13 @@ import { SpacesQueries } from '../../spaces/services/spaces-queries.service'import { USER_PERMISSION } from '../../users/constants/user'import { UserModel } from '../../users/models/user.model'import { FileProps } from '../interfaces/file-props.interface'-import { FileRecentLocation } from '../interfaces/file-recent-location.interface'-import { FileRecent } from '../schemas/file-recent.interface'+import type { FileRecent, FileRecentDeletion, FileRecentLocation, FileRecentUpdate } from '../schemas/file-recent.interface'+import { getProps } from '../utils/files'import { FilesQueries } from './files-queries.service'@Injectable()export class FilesRecents {- private readonly keepTime = '14d'+ private readonly keepTimeMs = convertHumanTimeToMs('14d')constructor(private readonly filesQueries: FilesQueries,@@ -35,42 +36,121 @@ export class FilesRecents {// Ignore trashed filesreturn}- const timestamp = currentTimeStamp(null, true) - convertHumanTimeToMs(this.keepTime)- const location = this.getLocation(user, space, files)- // only store files, ignore dirs- const fsRecents = files.filter((f) => !f.isDir && f.size > 0 && f.mtime > timestamp)+ const location = this.getSnapshotLocation(user, space, space.url, files)+ const recents = this.toRecents(space, location, files)+ if (space.inSharesList) {+ // The shares list is a complete snapshot and can expose the same file several times.+ await this.filesQueries.replaceRecents(location, recents)+ return+ }+ await this.updateSnapshot(location, recents)+ }++ async updateRecentFromEditor(user: UserModel, space: SpaceEnv, realPath: string): Promise<void> {+ if (space.inTrashRepository || space.inSharesList) return++ const file = await getProps(realPath, space.dbFile.path)+ if (!this.isRecentFile(file, this.getMinRecentMtime())) return++ // Match the identity produced while browsing: prefer the database id, otherwise keep the negative inode.+ file.id = (await this.filesQueries.getSpaceFileId(file, space.dbFile, { withDir: false })) ?? file.id+ const location: FileRecentLocation = {+ ...this.getRepositoryLocation(user.id, space.id, space.inPersonalSpace, space.inSharesRepository),+ path: path.dirname(space.url)+ }+ await this.filesQueries.upsertRecent(location, {+ id: file.id,+ name: file.name,+ mtime: file.mtime,+ mime: file.mime,+ ...location+ } as FileRecent)+ }++ private toRecents(space: SpaceEnv, location: FileRecentLocation, files: FileProps[]): FileRecent[] {+ const minMtime = this.getMinRecentMtime()+ // FileProps.id is the browser identity: database id when available, otherwise negative inode.+ const recents = new Map<number, FileRecent>()+ for (const file of files) {+ if (!this.isRecentFile(file, minMtime)) continue+ recents.set(file.id, {+ id: file.id,+ name: file.name,+ mtime: file.mtime,+ mime: file.mime,+ ...location,+ ...(space.inSharesList && { shareId: file.root.id })+ } as FileRecent)+ }+ return [...recents.values()]+ }++ private async updateSnapshot(location: FileRecentLocation, fsRecents: FileRecent[]): Promise<void> {const dbRecents = await this.filesQueries.getRecentsFromLocation(location)if (!fsRecents.length && !dbRecents.length) {return}- const [add, update, remove] = diffCollection(dbRecents as any, fsRecents as any, ['mtime', 'name'])- const toAdd: Partial<FileRecent>[] = add.map(- (f: FileProps): Partial<FileRecent> =>- ({- id: f.id,- name: f.name,- mtime: f.mtime,- mime: f.mime,- ...location,- ...(space.inSharesList && { shareId: f.root.id })- }) as FileRecent+ const dbRecentsById = new Map(dbRecents.map((recent) => [recent.id, recent]))+ const add: FileRecent[] = []+ const update: FileRecentUpdate[] = []+ for (const recent of fsRecents) {+ const dbRecent = dbRecentsById.get(recent.id)+ if (!dbRecent) {+ add.push(recent)+ continue+ }+ const changes: Omit<FileRecentUpdate, 'id'> = {}+ if (recent.name !== dbRecent.name) changes.name = recent.name+ if (recent.mtime !== dbRecent.mtime) changes.mtime = recent.mtime+ if (recent.mime !== dbRecent.mime) changes.mime = recent.mime+ if (Object.keys(changes).length) update.push({ id: recent.id, ...changes })+ dbRecentsById.delete(recent.id)+ }+ const remove = [...dbRecentsById.keys()]+ if (!add.length && !update.length && !remove.length) {+ return+ }+ await this.filesQueries.updateRecents(location, add, update, remove)+ }++ async deleteRecents(deletions: FileRecentDeletion[]): Promise<void> {+ await this.filesQueries.deleteRecents(+ deletions.map(({ userId, spaceId, inPersonalSpace, inSharesRepository, path: sourcePath }) => ({+ ...this.getRepositoryLocation(userId, spaceId, inPersonalSpace, inSharesRepository),+ path: path.normalize(sourcePath)+ })))- const toUpdate: Record<string | 'object', Partial<FileProps> | FileProps>[] = convertDiffUpdate(update)- const toRemove: number[] = remove.map((f: FileRecent) => f.id)- await this.filesQueries.updateRecents(location, toAdd, toUpdate, toRemove)}- private getLocation(user: UserModel, space: SpaceEnv, files: FileProps[]): FileRecentLocation {- const location: FileRecentLocation = { path: space.url }- if (space.inPersonalSpace) {- location.ownerId = user.id- } else if (space.inSharesList) {- location.shareId = files.map((f) => f.root.id)- } else if (space.inSharesRepository) {- location.shareId = space.id- } else {- location.spaceId = space.id+ private getMinRecentMtime(): number {+ // get the oldest modification time eligible for recents+ return currentTimeStamp(null, true) - this.keepTimeMs+ }++ private isRecentFile(file: FileProps, minMtime: number): boolean {+ // only keep non-empty files modified within the retention period+ return !file.isDir && file.size > 0 && file.mtime > minMtime+ }++ private getRepositoryLocation(+ userId: number,+ spaceId: number,+ inPersonalSpace: boolean,+ inSharesRepository: boolean+ ): Omit<FileRecentLocation, 'path'> {+ if (inPersonalSpace) {+ return { ownerId: userId }+ }+ if (inSharesRepository) {+ return { shareId: spaceId }+ }+ return { spaceId }+ }++ private getSnapshotLocation(user: UserModel, space: SpaceEnv, logicalPath: string, files: FileProps[]): FileRecentLocation {+ if (space.inSharesList) {+ return { path: logicalPath, shareId: files.map((file) => file.root.id) }}- return location+ return { ...this.getRepositoryLocation(user.id, space.id, space.inPersonalSpace, space.inSharesRepository), path: logicalPath }}}diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 4c3dd1c0..8d320856 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -125,8 +125,9 @@ export class FilesScheduler {DELETEFROM ${filesRecents}WHERE ${fk} IS NOT NULL- AND id NOT IN (SELECT id- FROM (SELECT id,+ AND (${fk}, ${filesRecents.id}) NOT IN (SELECT repositoryId, id+ FROM (SELECT ${fk} AS repositoryId,+ id,ROW_NUMBER() OVER (PARTITION BY ${fk} ORDER BY ${filesRecents.mtime} DESC) AS rnFROM ${filesRecents}WHERE ${fk} IS NOT NULL) AS ranked