Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): synchronize recent files after editor updates and deletions

johaven2b87607e871a
diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts
index 37958171..92b892e3 100644
--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts
+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts
@@ -9,8 +9,10 @@ import { ContextManager } from '../../../../infrastructure/context/services/cont
import { SPACE_OPERATION } from '../../../spaces/constants/spaces'
import type { SpaceEnv } from '../../../spaces/models/space-env.model'
import type { UserModel } from '../../../users/models/user.model'
+import { ACTION } from '../../../../common/constants'
import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'
import { FILE_MODE } from '../../constants/operations'
+import { FileEvent } from '../../events/file-events'
import { FileLockProps } from '../../interfaces/file-props.interface'
import { LockConflict } from '../../models/file-lock-error'
import { FilesLockManager } from '../../services/files-lock-manager.service'
@@ -338,6 +340,7 @@ describe(CollaboraOnlineManager.name, () => {
vi.spyOn(fs, 'stat').mockResolvedValue(mockStats as any)
vi.spyOn(filesUtils, 'fileSize').mockResolvedValue(1024)
+ const emitSpy = vi.spyOn(FileEvent, 'emit')
const result = await service.saveDocument(mockRequest)
@@ -347,6 +350,13 @@ describe(CollaboraOnlineManager.name, () => {
expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/tmp/document-unique.docx', mockRequest.raw)
expect(filesUtils.copyFileContent).toHaveBeenCalledWith('/tmp/document-unique.docx', mockSpace.realPath)
expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')
+ expect(emitSpy).toHaveBeenCalledWith('event', {
+ user: mockUser,
+ space: mockSpace,
+ action: ACTION.UPDATE,
+ rPath: mockSpace.realPath,
+ source: 'editor'
+ })
})
it('should throw error when document size mismatch', async () => {
diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts
index 1b8b72a3..ae35d09c 100644
--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts
+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts
@@ -136,7 +136,7 @@ export class CollaboraOnlineManager {
try {
await copyFileContent(tmpFilePath, req.space.realPath)
// emit file event
- FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath })
+ FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath, source: 'editor' })
await removeFiles(tmpFilePath)
const fStats = await fs.stat(req.space.realPath)
return { LastModifiedTime: fStats.mtime.toISOString() } satisfies CollaboraSaveDocumentDto
diff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts
index b01b951e..c4101010 100644
--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts
+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts
@@ -11,8 +11,10 @@ import { Cache } from '../../../../infrastructure/cache/cache.service'
import { ContextManager } from '../../../../infrastructure/context/services/context-manager.service'
import type { SpaceEnv } from '../../../spaces/models/space-env.model'
import type { UserModel } from '../../../users/models/user.model'
+import { ACTION } from '../../../../common/constants'
import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'
import { FILE_MODE } from '../../constants/operations'
+import { FileEvent } from '../../events/file-events'
import { LockConflict } from '../../models/file-lock-error'
import { FilesLockManager } from '../../services/files-lock-manager.service'
import * as filesUtils from '../../utils/files'
@@ -275,6 +277,7 @@ describe(OnlyOfficeManager.name, () => {
}
const expectSuccessfulSaveCallback = async (url: string, callbackData: Record<string, any> = {}) => {
+ const emitSpy = vi.spyOn(FileEvent, 'emit')
jwtService.verifyAsync.mockResolvedValue({
status: 2,
actions: [],
@@ -288,6 +291,13 @@ describe(OnlyOfficeManager.name, () => {
const result = await service.callBack(mockUser, mockSpaceEnv, mockToken)
expect(result).toEqual({ error: 0 })
+ expect(emitSpy).toHaveBeenCalledWith('event', {
+ user: mockUser,
+ space: mockSpaceEnv,
+ action: ACTION.UPDATE,
+ rPath: mockSpaceEnv.realPath,
+ source: 'editor'
+ })
expect(httpService.axiosRef).toHaveBeenCalledWith(expect.objectContaining({ url, maxRedirects: 0 }))
}
diff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.ts
index fd21d814..1e11b8c1 100644
--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts
+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.ts
@@ -384,7 +384,7 @@ export class OnlyOfficeManager {
try {
await copyFileContent(tmpFilePath, space.realPath)
// emit file event
- FileEvent.emit('event', { user: user, space: space, action: ACTION.UPDATE, rPath: space.realPath })
+ FileEvent.emit('event', { user: user, space: space, action: ACTION.UPDATE, rPath: space.realPath, source: 'editor' })
await removeFiles(tmpFilePath)
} catch (e) {
throw new Error(`unable to save document : ${e.message}`)
diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.ts
index 3061c6a4..60d448fb 100644
--- a/backend/src/applications/files/interfaces/file-event.interface.ts
+++ b/backend/src/applications/files/interfaces/file-event.interface.ts
@@ -11,6 +11,7 @@ export interface FileEventType {
space: SpaceEnv
action: ACTION
rPath: string
+ source?: 'editor'
}
export interface FileEventEmit {
diff --git a/backend/src/applications/files/interfaces/file-recent-location.interface.ts b/backend/src/applications/files/interfaces/file-recent-location.interface.ts
deleted file mode 100644
index 2d4acc9e..00000000
--- a/backend/src/applications/files/interfaces/file-recent-location.interface.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-export interface FileRecentLocation {
- ownerId?: number
- spaceId?: number
- shareId?: number | number[]
- path: string
-}
diff --git a/backend/src/applications/files/schemas/file-recent.interface.ts b/backend/src/applications/files/schemas/file-recent.interface.ts
index f165716f..984647f9 100644
--- a/backend/src/applications/files/schemas/file-recent.interface.ts
+++ b/backend/src/applications/files/schemas/file-recent.interface.ts
@@ -3,6 +3,7 @@ import { filesRecents } from './files-recents.schema'
type FileRecentSchema = typeof filesRecents.$inferSelect
export class FileRecent implements FileRecentSchema {
+ // Browser file identity: database id when available, otherwise negative inode.
id: number
ownerId: number
spaceId: number
@@ -12,3 +13,20 @@ export class FileRecent implements FileRecentSchema {
mime: string
mtime: number
}
+
+export interface FileRecentLocation {
+ ownerId?: number
+ spaceId?: number
+ shareId?: number | number[]
+ path: string
+}
+
+export type FileRecentUpdate = Pick<FileRecent, 'id'> & Partial<Pick<FileRecent, 'name' | 'mime' | 'mtime'>>
+
+export interface FileRecentDeletion {
+ userId: number
+ spaceId: number
+ inPersonalSpace: boolean
+ inSharesRepository: boolean
+ path: string
+}
diff --git a/backend/src/applications/files/schemas/files.schema.ts b/backend/src/applications/files/schemas/files.schema.ts
index 2b930a9f..19632db7 100644
--- a/backend/src/applications/files/schemas/files.schema.ts
+++ b/backend/src/applications/files/schemas/files.schema.ts
@@ -1,4 +1,4 @@
-import { SQL, sql } from 'drizzle-orm'
+import { SQL, sql, SQLWrapper } from 'drizzle-orm'
import { AnyMySqlColumn, bigint, boolean, index, mysqlTable, varchar } from 'drizzle-orm/mysql-core'
import { escapeSQLRegexp } from '../../../common/functions'
import { shares } from '../../shares/schemas/shares.schema'
@@ -46,7 +46,10 @@ export const files = mysqlTable(
// supports the case where path = '.' or './sync-in' and removes '.' or './' if exists
export const filePathSQL = (file: any): SQL<string> => sql`REGEXP_REPLACE(CONCAT(${file.path}, '/', ${file.name}), '^(\\\\.\\\\/){0,1}(.*)', '\\\\2')`
-export const childFilesFindRegexp = (path: string): SQL<string> => sql`${files.path}${sql.raw(`REGEXP '^${escapeSQLRegexp(path)}(/|$)'`)}`
+export const childPathFindRegexp = (pathSQL: SQLWrapper, path: string): SQL<string> =>
+ sql`${pathSQL}${sql.raw(`REGEXP '^${escapeSQLRegexp(path)}(/|$)'`)}`
+
+export const childFilesFindRegexp = (path: string): SQL<string> => childPathFindRegexp(files.path, path)
export const childFilesReplaceRegexp = (srcPath: string, dstPath: string): SQL<string> =>
sql`REGEXP_REPLACE(${files.path}, '^${sql.raw(escapeSQLRegexp(srcPath))}', '${sql.raw(escapeSQLRegexp(dstPath))}')`
diff --git a/backend/src/applications/files/services/files-event-manager.service.spec.ts b/backend/src/applications/files/services/files-event-manager.service.spec.ts
index ef11911d..b6c270fe 100644
--- a/backend/src/applications/files/services/files-event-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-event-manager.service.spec.ts
@@ -10,16 +10,27 @@ import { quotaCacheKeyFromSpace } from '../utils/quota'
import { indexingUpdateCacheKeysFromSpace } from '../utils/indexing'
import { configuration } from '../../../configuration/config.environment'
import type { Mock } from 'vitest'
+import { FilesRecents } from './files-recents.service'
describe(FilesEventManager.name, () => {
let service: FilesEventManager
let cacheSetMock: Mock<(key: string, value: unknown, ttl?: number) => Promise<boolean>>
+ let deleteRecentsMock: Mock
+ let updateRecentFromEditorMock: Mock
let contentIndexingEnabled: boolean
const buildEvent = (props?: Partial<FileEventType>): FileEventType =>
({
user: { id: 7, login: 'john' } as any,
- space: { id: 13, alias: 'project', inPersonalSpace: false, inSharesRepository: false, root: {} } as any,
+ space: {
+ id: 13,
+ alias: 'project',
+ url: 'files/project/document.bin',
+ inTrashRepository: false,
+ inPersonalSpace: false,
+ inSharesRepository: false,
+ root: {}
+ } as any,
action: ACTION.ADD,
rPath: '/files/document.bin',
...props
@@ -27,12 +38,18 @@ describe(FilesEventManager.name, () => {
beforeEach(async () => {
cacheSetMock = vi.fn().mockResolvedValue(true)
+ deleteRecentsMock = vi.fn().mockResolvedValue(undefined)
+ updateRecentFromEditorMock = vi.fn().mockResolvedValue(undefined)
const module: TestingModule = await Test.createTestingModule({
providers: [
FilesEventManager,
{
provide: Cache,
useValue: { set: cacheSetMock }
+ },
+ {
+ provide: FilesRecents,
+ useValue: { deleteRecents: deleteRecentsMock, updateRecentFromEditor: updateRecentFromEditorMock }
}
]
}).compile()
@@ -44,13 +61,8 @@ describe(FilesEventManager.name, () => {
afterEach(async () => {
configuration.applications.files.contentIndexing.enabled = contentIndexingEnabled
- vi.useRealTimers()
- vi.clearAllMocks()
await service.onModuleDestroy()
- })
-
- it('should be defined', () => {
- expect(service).toBeDefined()
+ vi.restoreAllMocks()
})
it('should cache a quota key once for duplicate non-delete events', async () => {
@@ -62,17 +74,60 @@ describe(FilesEventManager.name, () => {
expect(cacheSetMock).toHaveBeenCalledWith(quotaCacheKeyFromSpace(event.user.id, event.space, true), true, CACHE_QUOTA_TTL)
})
+ it('should delete recents before processing only the latest editor update per path', async () => {
+ const callOrder: string[] = []
+ deleteRecentsMock.mockImplementationOnce(async () => {
+ callOrder.push('delete')
+ })
+ updateRecentFromEditorMock.mockImplementationOnce(async () => {
+ callOrder.push('update')
+ })
+ const first = buildEvent({ action: ACTION.UPDATE, source: 'editor', rPath: '/files/first.bin' })
+ const latest = { ...first, rPath: '/files/latest.bin' }
+
+ await (service as any).processEvents([first, buildEvent({ action: ACTION.UPDATE }), buildEvent({ action: ACTION.DELETE }), latest])
+
+ expect(deleteRecentsMock).toHaveBeenCalledWith([
+ { userId: 7, spaceId: 13, inPersonalSpace: false, inSharesRepository: false, path: 'files/project/document.bin' }
+ ])
+ expect(updateRecentFromEditorMock).toHaveBeenCalledOnce()
+ expect(updateRecentFromEditorMock).toHaveBeenCalledWith(latest.user, latest.space, latest.rPath)
+ expect(callOrder).toEqual(['delete', 'update'])
+ })
+
+ it('should ignore recent deletion errors', async () => {
+ deleteRecentsMock.mockRejectedValueOnce(new Error('database unavailable'))
+
+ await expect((service as any).processEvents([buildEvent({ action: ACTION.DELETE })])).resolves.toBeUndefined()
+ })
+
it('should ignore quota cache update for delete events', async () => {
await (service as any).processEvents([buildEvent({ action: ACTION.DELETE, rPath: '/files/deleted.tmp' })])
expect(cacheSetMock).not.toHaveBeenCalled()
})
+ it('should ignore deletes from trash and shares list for recents', async () => {
+ const event = buildEvent()
+ await (service as any).processEvents([
+ buildEvent({ action: ACTION.DELETE, space: { ...event.space, inTrashRepository: true } as any }),
+ buildEvent({ action: ACTION.DELETE, space: { ...event.space, inSharesList: true } as any })
+ ])
+
+ expect(deleteRecentsMock).not.toHaveBeenCalled()
+ })
+
it('should cache quota and indexing keys for indexable files', async () => {
configuration.applications.files.contentIndexing.enabled = true
const event = buildEvent({
rPath: '/shares/spec.pdf',
- space: { id: 42, alias: 'shared-space', inPersonalSpace: false, inSharesRepository: true, root: {} } as any
+ space: {
+ id: 42,
+ alias: 'shared-space',
+ inPersonalSpace: false,
+ inSharesRepository: true,
+ root: {}
+ } as any
})
const quotaKey = quotaCacheKeyFromSpace(event.user.id, event.space, true)
const indexingKeys = indexingUpdateCacheKeysFromSpace(event.user.id, event.space)
diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.ts
index 8520b835..33abe4c9 100644
--- a/backend/src/applications/files/services/files-event-manager.service.ts
+++ b/backend/src/applications/files/services/files-event-manager.service.ts
@@ -9,10 +9,12 @@ import { getExtensionWithoutDot } from '../utils/files'
import { CACHE_INDEXING_EVENT_TTL, INDEXABLE_EXTENSIONS } from '../constants/indexing'
import { indexingUpdateCacheKeysFromSpace } from '../utils/indexing'
import { configuration } from '../../../configuration/config.environment'
+import { FilesRecents } from './files-recents.service'
@Injectable()
export class FilesEventManager implements OnModuleDestroy {
/* Used to:
+ - reconcile recents after deletions and editor saves
- store cached events for storage usage updates
- store indexing events for full-text index updates
- todo: handle versioning
@@ -27,7 +29,10 @@ export class FilesEventManager implements OnModuleDestroy {
private isFlushing = false
private flushRequested = false
- constructor(private readonly cache: Cache) {
+ constructor(
+ private readonly cache: Cache,
+ private readonly filesRecents: FilesRecents
+ ) {
FileEvent.on('event', this.onFileEvent)
}
@@ -50,6 +55,15 @@ export class FilesEventManager implements OnModuleDestroy {
this.startFlushTimer()
}
+ private async processRecentEditorUpdate(event: FileEventType): Promise<void> {
+ try {
+ await this.filesRecents.updateRecentFromEditor(event.user, event.space, event.rPath)
+ } catch (e) {
+ if ((e as NodeJS.ErrnoException)?.code === 'ENOENT') return
+ this.logger.warn({ tag: this.processRecentEditorUpdate.name, msg: `Could not update recent file ${event.space.url}: ${e}` })
+ }
+ }
+
private startFlushTimer(): void {
if (this.flushTimer) {
return
@@ -106,7 +120,45 @@ export class FilesEventManager implements OnModuleDestroy {
this.logger.warn({ tag: this.processEvents.name, msg: `Could not process event: ${JSON.stringify(event)} - ${e}` })
}
}
- await this.storeEventsInCache()
+ await Promise.all([this.storeEventsInCache(), this.processRecentEvents(fEvents)])
+ }
+
+ private async processRecentEvents(events: FileEventType[]): Promise<void> {
+ // Delete first, then let editor updates reinsert only files that still exist on disk.
+ await this.processRecentDeletes(events)
+ await this.processRecentEditorUpdates(events)
+ }
+
+ private async processRecentEditorUpdates(events: FileEventType[]): Promise<void> {
+ const updatesByPath = new Map<string, FileEventType>()
+ for (const event of events) {
+ // Only editor saves update recents from events; other updates are reconciled while browsing.
+ if (event.action !== ACTION.UPDATE || event.source !== 'editor') continue
+ const key = `${event.user.id}:${event.space.id}:${event.space.inPersonalSpace}:${event.space.inSharesRepository}:${event.space.url}`
+ // Keep the latest save per logical path to perform at most one stat and upsert per buffered batch.
+ updatesByPath.set(key, event)
+ }
+ await Promise.all([...updatesByPath.values()].map((event) => this.processRecentEditorUpdate(event)))
+ }
+
+ private async processRecentDeletes(events: FileEventType[]): Promise<void> {
+ const deletions = events
+ .filter(
+ ({ action, space }) => (action === ACTION.DELETE || action === ACTION.DELETE_PERMANENTLY) && !space.inTrashRepository && !space.inSharesList
+ )
+ .map(({ user, space }) => ({
+ userId: user.id,
+ spaceId: space.id,
+ inPersonalSpace: space.inPersonalSpace,
+ inSharesRepository: space.inSharesRepository,
+ path: space.url
+ }))
+ if (!deletions.length) return
+ try {
+ await this.filesRecents.deleteRecents(deletions)
+ } catch (e) {
+ this.logger.warn({ tag: this.processRecentDeletes.name, msg: `Could not delete recent files: ${e}` })
+ }
}
private async storeEventsInCache() {
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 50dc68d1..0c360f9f 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -365,7 +365,10 @@ describe(FilesManager.name, () => {
expect(tmpWritePath.endsWith('-report.txt')).toBe(true)
expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, expect.anything())
expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, '/data/users/john/files/report.txt', true)
- expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.UPDATE, rPath: '/data/users/john/files/report.txt' }))
+ expect(emitSpy).toHaveBeenCalledWith(
+ 'event',
+ expect.objectContaining({ action: ACTION.UPDATE, rPath: '/data/users/john/files/report.txt', source: 'editor' })
+ )
})
it('should reject PATCH when destination does not exist', async () => {
@@ -416,6 +419,7 @@ describe(FilesManager.name, () => {
expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, space.realPath, true)
expect(filesUtils.removeFiles).not.toHaveBeenCalled()
expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.UPDATE, rPath: space.realPath }))
+ expect(emitSpy).not.toHaveBeenCalledWith('event', expect.objectContaining({ source: 'editor' }))
})
it('should create missing destination directory and release created lock for POST nested upload', async () => {
@@ -1013,11 +1017,13 @@ describe(FilesManager.name, () => {
})
const trashFile = '/data/users/john/trash/documents/document.txt'
prepareFileTransfer(space.realPath, trashFile)
+ const emitSpy = vi.spyOn(FileEvent, 'emit')
await service.delete(user, space)
expect(filesUtils.moveFiles).toHaveBeenCalledWith(space.realPath, trashFile, true)
expect(filesTasksTransfer.delete).not.toHaveBeenCalled()
+ expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.DELETE, rPath: trashFile })
})
})
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index a7543766..0f00fd50 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -306,7 +306,13 @@ export class FilesManager {
if (fileWritten) {
// Emit only after the final destination has been written or moved into place.
const fileEventAction: ACTION = patchMethod || (dstExists && !dstIsDir) ? ACTION.UPDATE : ACTION.ADD
- FileEvent.emit('event', { user, space, action: fileEventAction, rPath: dstFile })
+ FileEvent.emit('event', {
+ user,
+ space,
+ action: fileEventAction,
+ rPath: dstFile,
+ ...(patchMethod && { source: 'editor' as const })
+ })
}
if (!patchMethod && created) {
// Remove the file lock only if it has not been refreshed
@@ -557,6 +563,10 @@ export class FilesManager {
const userSpace = new SpaceEnv(SPACE_PERSONAL, null, false)
userSpace.setup(user, SPACE_REPOSITORY.TRASH, null, [], [])
FileEvent.emit('event', { user, space: userSpace, action: ACTION.ADD, rPath: trashFile })
+ } else {
+ // emit an event for the file or directory moved to the trash
+ // space keeps its original path and rPath is its new trash path
+ FileEvent.emit('event', { user, space, action: ACTION.DELETE, rPath: trashFile })
}
} else {
// unsupported case: delete the file (this shouldn't happen)
diff --git a/backend/src/applications/files/services/files-queries.service.ts b/backend/src/applications/files/services/files-queries.service.ts
index 66525039..9cb34502 100644
--- a/backend/src/applications/files/services/files-queries.service.ts
+++ b/backend/src/applications/files/services/files-queries.service.ts
@@ -1,6 +1,5 @@
import { Inject, Injectable, Logger } from '@nestjs/common'
import { and, desc, eq, getTableColumns, inArray, isNull, or, SelectedFields, sql, SQL } from 'drizzle-orm'
-import { popFromObject } from '../../../common/shared'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
import { concatDistinctObjectsInArray, convertToWhere, dbCheckAffectedRows, dbGetInsertedId } from '../../../infrastructure/database/utils'
@@ -12,12 +11,11 @@ import { syncClients } from '../../sync/schemas/sync-clients.schema'
import { syncPaths } from '../../sync/schemas/sync-paths.schema'
import { FileDBProps } from '../interfaces/file-db-props.interface'
import { FileProps } from '../interfaces/file-props.interface'
-import { FileRecentLocation } from '../interfaces/file-recent-location.interface'
-import { FileRecent } from '../schemas/file-recent.interface'
+import type { FileRecent, FileRecentLocation, FileRecentUpdate } from '../schemas/file-recent.interface'
import { File } from '../schemas/file.interface'
import { filesRecents } from '../schemas/files-recents.schema'
-import { childFilesFindRegexp, childFilesReplaceRegexp, filePathSQL, files } from '../schemas/files.schema'
-import { dirName, fileName } from '../utils/files'
+import { childFilesFindRegexp, childFilesReplaceRegexp, childPathFindRegexp, filePathSQL, files } from '../schemas/files.schema'
+import { dirName, fileName, isPathInside } from '../utils/files'
@Injectable()
export class FilesQueries {
@@ -308,7 +306,7 @@ export class FilesQueries {
.limit(limit)
}
- getRecentsFromLocation(location: Partial<Record<keyof FileRecent, any>>): Promise<FileRecent[]> {
+ getRecentsFromLocation(location: FileRecentLocation): Promise<FileRecent[]> {
const where: SQL[] = convertToWhere(filesRecents, location)
return this.db
.select(getTableColumns(filesRecents))
@@ -316,31 +314,73 @@ export class FilesQueries {
.where(and(...where))
}
- async updateRecents(
- location: FileRecentLocation,
- add: Partial<FileRecent>[] | FileRecent[],
- update: Record<string | 'object', Partial<FileProps> | FileProps>[],
- remove: FileRecent['id'][]
- ): Promise<void> {
+ async deleteRecents(locations: FileRecentLocation[]): Promise<void> {
+ const groups = new Map<string, { repository: Omit<FileRecentLocation, 'path'>; paths: Set<string> }>()
+ for (const { path: sourcePath, ...repository } of locations) {
+ const key = JSON.stringify(repository)
+ const group = groups.get(key)
+ if (group) {
+ group.paths.add(sourcePath)
+ } else {
+ groups.set(key, { repository, paths: new Set([sourcePath]) })
+ }
+ }
+
+ const batchSize = 50
+ for (const { repository, paths } of groups.values()) {
+ const compactPaths: string[] = []
+ for (const sourcePath of [...paths].sort((a, b) => a.length - b.length)) {
+ if (!compactPaths.some((parentPath) => isPathInside(parentPath, sourcePath, true))) compactPaths.push(sourcePath)
+ }
+ const where: SQL[] = convertToWhere(filesRecents, repository)
+ for (let offset = 0; offset < compactPaths.length; offset += batchSize) {
+ const batch = compactPaths.slice(offset, offset + batchSize)
+ try {
+ await this.db
+ .delete(filesRecents)
+ .where(and(...where, or(...batch.map((sourcePath) => childPathFindRegexp(filePathSQL(filesRecents), sourcePath)))))
+ } catch (e) {
+ this.logger.error({ tag: this.deleteRecents.name, msg: `${e}` })
+ }
+ }
+ }
+ }
+
+ async replaceRecents(location: FileRecentLocation, recents: FileRecent[]): Promise<void> {
+ const where: SQL[] = convertToWhere(filesRecents, location)
+ await this.db.transaction(async (tx) => {
+ await tx.delete(filesRecents).where(and(...where))
+ if (recents.length) await tx.insert(filesRecents).values(recents)
+ })
+ }
+
+ async upsertRecent(location: FileRecentLocation, recent: FileRecent): Promise<void> {
+ const where: SQL[] = convertToWhere(filesRecents, location)
+ await this.db.transaction(async (tx) => {
+ // The table has no unique key: replacing by logical location also removes a stale inode identity.
+ await tx.delete(filesRecents).where(and(...where, eq(filesRecents.name, recent.name)))
+ await tx.insert(filesRecents).values(recent)
+ })
+ }
+
+ async updateRecents(location: FileRecentLocation, add: FileRecent[], update: FileRecentUpdate[], remove: FileRecent['id'][]): Promise<void> {
const where: SQL[] = convertToWhere(filesRecents, location)
// add
if (add.length) {
try {
- await this.db.insert(filesRecents).values(add as FileRecent[])
+ await this.db.insert(filesRecents).values(add)
} catch (e) {
this.logger.error({ tag: this.updateRecents.name, msg: `${e}` })
}
}
// update
if (update.length) {
- for (const props of update) {
- const f: FileProps = popFromObject('object', props)
+ for (const { id, ...props } of update) {
try {
await this.db
.update(filesRecents)
- .set({ ...props })
- .where(and(...where, eq(filesRecents.id, f.id)))
- .limit(1)
+ .set(props)
+ .where(and(...where, eq(filesRecents.id, id)))
} catch (e) {
this.logger.error({ tag: this.updateRecents.name, msg: `${e}` })
}
diff --git a/backend/src/applications/files/services/files-recents.service.spec.ts b/backend/src/applications/files/services/files-recents.service.spec.ts
index b71289b3..d94b30be 100644
--- a/backend/src/applications/files/services/files-recents.service.spec.ts
+++ b/backend/src/applications/files/services/files-recents.service.spec.ts
@@ -6,14 +6,19 @@ import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
import { USER_PERMISSION } from '../../users/constants/user'
import { FilesQueries } from './files-queries.service'
import { FilesRecents } from './files-recents.service'
-import { Mock } from 'vitest'
+import type { Mock } from 'vitest'
+import * as filesUtils from '../utils/files'
describe(FilesRecents.name, () => {
let service: FilesRecents
let filesQueries: {
getRecentsFromUser: Mock
getRecentsFromLocation: Mock
+ deleteRecents: Mock
+ replaceRecents: Mock
updateRecents: Mock
+ getSpaceFileId: Mock
+ upsertRecent: Mock
}
let spacesQueries: {
spaceIds: Mock
@@ -26,7 +31,11 @@ describe(FilesRecents.name, () => {
filesQueries = {
getRecentsFromUser: vi.fn().mockResolvedValue([]),
getRecentsFromLocation: vi.fn().mockResolvedValue([]),
- updateRecents: vi.fn().mockResolvedValue(undefined)
+ deleteRecents: vi.fn().mockResolvedValue(undefined),
+ replaceRecents: vi.fn().mockResolvedValue(undefined),
+ updateRecents: vi.fn().mockResolvedValue(undefined),
+ getSpaceFileId: vi.fn().mockResolvedValue(undefined),
+ upsertRecent: vi.fn().mockResolvedValue(undefined)
}
spacesQueries = {
spaceIds: vi.fn().mockResolvedValue([])
@@ -51,7 +60,7 @@ describe(FilesRecents.name, () => {
})
afterEach(() => {
- vi.clearAllMocks()
+ vi.restoreAllMocks()
})
const userWithPermissions = (applications: USER_PERMISSION[] = [], props: Record<string, any> = {}) =>
@@ -63,10 +72,6 @@ describe(FilesRecents.name, () => {
...props
}) as any
- it('should be defined', () => {
- expect(service).toBeDefined()
- })
-
it('should load recents from user accessible spaces and shares', async () => {
const recents = [{ id: 1, name: 'a.txt' }]
spacesQueries.spaceIds.mockResolvedValueOnce([10, 11])
@@ -127,6 +132,85 @@ describe(FilesRecents.name, () => {
expect(filesQueries.updateRecents).not.toHaveBeenCalled()
})
+ it('should normalize and map recent deletions to their repositories', async () => {
+ await service.deleteRecents([
+ { userId: 5, spaceId: 0, inPersonalSpace: true, inSharesRepository: false, path: 'files/personal/docs/../old.txt' },
+ { userId: 5, spaceId: 9, inPersonalSpace: false, inSharesRepository: false, path: 'files/project/archive' },
+ { userId: 5, spaceId: 12, inPersonalSpace: false, inSharesRepository: true, path: 'shares/project/report.pdf' }
+ ])
+
+ expect(filesQueries.deleteRecents).toHaveBeenCalledWith([
+ { ownerId: 5, path: 'files/personal/old.txt' },
+ { spaceId: 9, path: 'files/project/archive' },
+ { shareId: 12, path: 'shares/project/report.pdf' }
+ ])
+ })
+
+ it('should stat and upsert one recent file after an editor update', async () => {
+ const now = currentTimeStamp(null, true)
+ const file = {
+ id: -123,
+ path: 'docs',
+ name: 'report.txt',
+ isDir: false,
+ size: 42,
+ ctime: now,
+ mtime: now,
+ mime: 'text/plain'
+ }
+ const user = { id: 5 } as any
+ const space = {
+ id: 9,
+ url: 'files/project/docs/report.txt',
+ dbFile: { spaceId: 9, path: 'docs/report.txt', inTrash: false },
+ inTrashRepository: false,
+ inSharesList: false,
+ inPersonalSpace: false,
+ inSharesRepository: false
+ } as any
+ vi.spyOn(filesUtils, 'getProps').mockResolvedValueOnce(file as any)
+ filesQueries.getSpaceFileId.mockResolvedValueOnce(321)
+
+ await service.updateRecentFromEditor(user, space, '/data/project/docs/report.txt')
+
+ expect(filesUtils.getProps).toHaveBeenCalledWith('/data/project/docs/report.txt', 'docs/report.txt')
+ expect(filesQueries.getSpaceFileId).toHaveBeenCalledWith(file, space.dbFile, { withDir: false })
+ expect(filesQueries.upsertRecent).toHaveBeenCalledWith(
+ { spaceId: 9, path: 'files/project/docs' },
+ { id: 321, spaceId: 9, path: 'files/project/docs', name: 'report.txt', mtime: now, mime: 'text/plain' }
+ )
+ })
+
+ it('should ignore an old file after an editor update', async () => {
+ vi.spyOn(filesUtils, 'getProps').mockResolvedValueOnce({
+ id: -123,
+ path: 'docs',
+ name: 'old.txt',
+ isDir: false,
+ size: 42,
+ ctime: 0,
+ mtime: currentTimeStamp(null, true) - convertHumanTimeToMs('30d'),
+ mime: 'text/plain'
+ } as any)
+
+ await service.updateRecentFromEditor(
+ { id: 5 } as any,
+ {
+ id: 0,
+ url: 'files/personal/docs/old.txt',
+ dbFile: { ownerId: 5, path: 'docs/old.txt', inTrash: false },
+ inTrashRepository: false,
+ inSharesList: false,
+ inPersonalSpace: true,
+ inSharesRepository: false
+ } as any,
+ '/data/users/john/files/docs/old.txt'
+ )
+
+ expect(filesQueries.getSpaceFileId).not.toHaveBeenCalled()
+ expect(filesQueries.upsertRecent).not.toHaveBeenCalled()
+ })
+
it('should skip persistence when filtered fs recents and db recents are both empty', async () => {
const oldMtime = currentTimeStamp(null, true) - convertHumanTimeToMs('30d')
const files = [
@@ -145,12 +229,12 @@ describe(FilesRecents.name, () => {
const now = currentTimeStamp(null, true)
const location = { ownerId: 5, path: 'files/personal/docs' }
const dbRecents = [
- { id: 1, name: 'old-name.txt', mtime: now - 1000, mime: 'text-plain' },
- { id: 2, name: 'remove-me.txt', mtime: now - 1000, mime: 'text-plain' }
+ { id: 101, name: 'old-name.txt', mtime: now - 1000, mime: 'text-plain', ...location },
+ { id: 102, name: 'remove-me.txt', mtime: now - 1000, mime: 'text-plain', ...location }
]
const fsFiles = [
- { id: 1, name: 'new-name.txt', isDir: false, size: 100, mtime: now, mime: 'text-plain' },
- { id: 3, name: 'add-me.txt', isDir: false, size: 200, mtime: now, mime: 'text-plain' }
+ { id: 101, name: 'new-name.txt', isDir: false, size: 100, mtime: now, mime: 'application-pdf' },
+ { id: 103, name: 'add-me.txt', isDir: false, size: 200, mtime: now, mime: 'text-plain' }
]
filesQueries.getRecentsFromLocation.mockResolvedValueOnce(dbRecents)
@@ -158,38 +242,31 @@ describe(FilesRecents.name, () => {
expect(filesQueries.getRecentsFromLocation).toHaveBeenCalledWith(location)
expect(filesQueries.updateRecents).toHaveBeenCalledTimes(1)
- const [loc, toAdd, toUpdate, toRemove] = filesQueries.updateRecents.mock.calls[0]
- expect(loc).toEqual(location)
- expect(toAdd).toEqual([{ id: 3, name: 'add-me.txt', mtime: now, mime: 'text-plain', ...location }])
- expect(toUpdate).toEqual([{ name: 'new-name.txt', mtime: now, object: fsFiles[0] }])
- expect(toRemove).toEqual([2])
+ const [actualLocation, toAdd, toUpdate, toRemove] = filesQueries.updateRecents.mock.calls[0]
+ expect(actualLocation).toEqual(location)
+ expect(toAdd).toEqual([{ id: 103, name: 'add-me.txt', mtime: now, mime: 'text-plain', ...location }])
+ expect(toUpdate).toEqual([{ id: 101, name: 'new-name.txt', mtime: now, mime: 'application-pdf' }])
+ expect(toRemove).toEqual([102])
})
- it('should use share location list and per-file share id when browsing shares list', async () => {
+ it('should replace the shares snapshot and deduplicate by file id', async () => {
const now = currentTimeStamp(null, true)
+ const space = { inTrashRepository: false, inSharesList: true, inSharesRepository: false, inPersonalSpace: false, id: 0, url: 'shares' } as any
const files = [
- { id: 10, name: 'a.txt', isDir: false, size: 5, mtime: now, mime: 'text-plain', root: { id: 101 } },
- { id: 11, name: 'b.txt', isDir: false, size: 5, mtime: now, mime: 'text-plain', root: { id: 102 } }
+ { id: 10, name: 'report.pdf', isDir: false, size: 5, mtime: now, mime: 'application-pdf', root: { id: 101 } },
+ { id: 10, name: 'report.pdf', isDir: false, size: 5, mtime: now, mime: 'application-pdf', root: { id: 102 } }
]
- filesQueries.getRecentsFromLocation.mockResolvedValueOnce([])
+ await service.updateRecents({ id: 5 } as any, space, files as any)
- await service.updateRecents(
- { id: 5 } as any,
- { inTrashRepository: false, inSharesList: true, inSharesRepository: false, inPersonalSpace: false, id: 0, url: 'shares' } as any,
- files as any
- )
-
- const [location, toAdd] = filesQueries.updateRecents.mock.calls[0]
- expect(location).toEqual({ shareId: [101, 102], path: 'shares' })
- expect(toAdd).toEqual([
- { id: 10, name: 'a.txt', mtime: now, mime: 'text-plain', path: 'shares', shareId: 101 },
- { id: 11, name: 'b.txt', mtime: now, mime: 'text-plain', path: 'shares', shareId: 102 }
+ expect(filesQueries.getRecentsFromLocation).not.toHaveBeenCalled()
+ expect(filesQueries.updateRecents).not.toHaveBeenCalled()
+ expect(filesQueries.replaceRecents).toHaveBeenCalledWith({ shareId: [101, 102], path: 'shares' }, [
+ { id: 10, name: 'report.pdf', mtime: now, mime: 'application-pdf', path: 'shares', shareId: 102 }
])
})
it('should use repository share id for a single share location', async () => {
const now = currentTimeStamp(null, true)
- filesQueries.getRecentsFromLocation.mockResolvedValueOnce([])
await service.updateRecents(
{ id: 5 } as any,
@@ -202,7 +279,6 @@ describe(FilesRecents.name, () => {
it('should use space id location for regular space repository', async () => {
const now = currentTimeStamp(null, true)
- filesQueries.getRecentsFromLocation.mockResolvedValueOnce([])
await service.updateRecents(
{ id: 5 } as any,
diff --git a/backend/src/applications/files/services/files-recents.service.ts b/backend/src/applications/files/services/files-recents.service.ts
index eaf6a8eb..534696e3 100644
--- a/backend/src/applications/files/services/files-recents.service.ts
+++ b/backend/src/applications/files/services/files-recents.service.ts
@@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common'
-import { convertDiffUpdate, convertHumanTimeToMs, diffCollection } from '../../../common/functions'
+import path from 'node:path'
+import { convertHumanTimeToMs } from '../../../common/functions'
import { currentTimeStamp } from '../../../common/shared'
import { SharesQueries } from '../../shares/services/shares-queries.service'
import { SpaceEnv } from '../../spaces/models/space-env.model'
@@ -7,13 +8,13 @@ import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
import { USER_PERMISSION } from '../../users/constants/user'
import { UserModel } from '../../users/models/user.model'
import { FileProps } from '../interfaces/file-props.interface'
-import { FileRecentLocation } from '../interfaces/file-recent-location.interface'
-import { FileRecent } from '../schemas/file-recent.interface'
+import type { FileRecent, FileRecentDeletion, FileRecentLocation, FileRecentUpdate } from '../schemas/file-recent.interface'
+import { getProps } from '../utils/files'
import { FilesQueries } from './files-queries.service'
@Injectable()
export class FilesRecents {
- private readonly keepTime = '14d'
+ private readonly keepTimeMs = convertHumanTimeToMs('14d')
constructor(
private readonly filesQueries: FilesQueries,
@@ -35,42 +36,121 @@ export class FilesRecents {
// Ignore trashed files
return
}
- const timestamp = currentTimeStamp(null, true) - convertHumanTimeToMs(this.keepTime)
- const location = this.getLocation(user, space, files)
- // only store files, ignore dirs
- const fsRecents = files.filter((f) => !f.isDir && f.size > 0 && f.mtime > timestamp)
+ const location = this.getSnapshotLocation(user, space, space.url, files)
+ const recents = this.toRecents(space, location, files)
+ if (space.inSharesList) {
+ // The shares list is a complete snapshot and can expose the same file several times.
+ await this.filesQueries.replaceRecents(location, recents)
+ return
+ }
+ await this.updateSnapshot(location, recents)
+ }
+
+ async updateRecentFromEditor(user: UserModel, space: SpaceEnv, realPath: string): Promise<void> {
+ if (space.inTrashRepository || space.inSharesList) return
+
+ const file = await getProps(realPath, space.dbFile.path)
+ if (!this.isRecentFile(file, this.getMinRecentMtime())) return
+
+ // Match the identity produced while browsing: prefer the database id, otherwise keep the negative inode.
+ file.id = (await this.filesQueries.getSpaceFileId(file, space.dbFile, { withDir: false })) ?? file.id
+ const location: FileRecentLocation = {
+ ...this.getRepositoryLocation(user.id, space.id, space.inPersonalSpace, space.inSharesRepository),
+ path: path.dirname(space.url)
+ }
+ await this.filesQueries.upsertRecent(location, {
+ id: file.id,
+ name: file.name,
+ mtime: file.mtime,
+ mime: file.mime,
+ ...location
+ } as FileRecent)
+ }
+
+ private toRecents(space: SpaceEnv, location: FileRecentLocation, files: FileProps[]): FileRecent[] {
+ const minMtime = this.getMinRecentMtime()
+ // FileProps.id is the browser identity: database id when available, otherwise negative inode.
+ const recents = new Map<number, FileRecent>()
+ for (const file of files) {
+ if (!this.isRecentFile(file, minMtime)) continue
+ recents.set(file.id, {
+ id: file.id,
+ name: file.name,
+ mtime: file.mtime,
+ mime: file.mime,
+ ...location,
+ ...(space.inSharesList && { shareId: file.root.id })
+ } as FileRecent)
+ }
+ return [...recents.values()]
+ }
+
+ private async updateSnapshot(location: FileRecentLocation, fsRecents: FileRecent[]): Promise<void> {
const dbRecents = await this.filesQueries.getRecentsFromLocation(location)
if (!fsRecents.length && !dbRecents.length) {
return
}
- const [add, update, remove] = diffCollection(dbRecents as any, fsRecents as any, ['mtime', 'name'])
- const toAdd: Partial<FileRecent>[] = add.map(
- (f: FileProps): Partial<FileRecent> =>
- ({
- id: f.id,
- name: f.name,
- mtime: f.mtime,
- mime: f.mime,
- ...location,
- ...(space.inSharesList && { shareId: f.root.id })
- }) as FileRecent
+ const dbRecentsById = new Map(dbRecents.map((recent) => [recent.id, recent]))
+ const add: FileRecent[] = []
+ const update: FileRecentUpdate[] = []
+ for (const recent of fsRecents) {
+ const dbRecent = dbRecentsById.get(recent.id)
+ if (!dbRecent) {
+ add.push(recent)
+ continue
+ }
+ const changes: Omit<FileRecentUpdate, 'id'> = {}
+ if (recent.name !== dbRecent.name) changes.name = recent.name
+ if (recent.mtime !== dbRecent.mtime) changes.mtime = recent.mtime
+ if (recent.mime !== dbRecent.mime) changes.mime = recent.mime
+ if (Object.keys(changes).length) update.push({ id: recent.id, ...changes })
+ dbRecentsById.delete(recent.id)
+ }
+ const remove = [...dbRecentsById.keys()]
+ if (!add.length && !update.length && !remove.length) {
+ return
+ }
+ await this.filesQueries.updateRecents(location, add, update, remove)
+ }
+
+ async deleteRecents(deletions: FileRecentDeletion[]): Promise<void> {
+ await this.filesQueries.deleteRecents(
+ deletions.map(({ userId, spaceId, inPersonalSpace, inSharesRepository, path: sourcePath }) => ({
+ ...this.getRepositoryLocation(userId, spaceId, inPersonalSpace, inSharesRepository),
+ path: path.normalize(sourcePath)
+ }))
)
- const toUpdate: Record<string | 'object', Partial<FileProps> | FileProps>[] = convertDiffUpdate(update)
- const toRemove: number[] = remove.map((f: FileRecent) => f.id)
- await this.filesQueries.updateRecents(location, toAdd, toUpdate, toRemove)
}
- private getLocation(user: UserModel, space: SpaceEnv, files: FileProps[]): FileRecentLocation {
- const location: FileRecentLocation = { path: space.url }
- if (space.inPersonalSpace) {
- location.ownerId = user.id
- } else if (space.inSharesList) {
- location.shareId = files.map((f) => f.root.id)
- } else if (space.inSharesRepository) {
- location.shareId = space.id
- } else {
- location.spaceId = space.id
+ private getMinRecentMtime(): number {
+ // get the oldest modification time eligible for recents
+ return currentTimeStamp(null, true) - this.keepTimeMs
+ }
+
+ private isRecentFile(file: FileProps, minMtime: number): boolean {
+ // only keep non-empty files modified within the retention period
+ return !file.isDir && file.size > 0 && file.mtime > minMtime
+ }
+
+ private getRepositoryLocation(
+ userId: number,
+ spaceId: number,
+ inPersonalSpace: boolean,
+ inSharesRepository: boolean
+ ): Omit<FileRecentLocation, 'path'> {
+ if (inPersonalSpace) {
+ return { ownerId: userId }
+ }
+ if (inSharesRepository) {
+ return { shareId: spaceId }
+ }
+ return { spaceId }
+ }
+
+ private getSnapshotLocation(user: UserModel, space: SpaceEnv, logicalPath: string, files: FileProps[]): FileRecentLocation {
+ if (space.inSharesList) {
+ return { path: logicalPath, shareId: files.map((file) => file.root.id) }
}
- return location
+ return { ...this.getRepositoryLocation(user.id, space.id, space.inPersonalSpace, space.inSharesRepository), path: logicalPath }
}
}
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 4c3dd1c0..8d320856 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -125,8 +125,9 @@ export class FilesScheduler {
DELETE
FROM ${filesRecents}
WHERE ${fk} IS NOT NULL
- AND id NOT IN (SELECT id
- FROM (SELECT id,
+ AND (${fk}, ${filesRecents.id}) NOT IN (SELECT repositoryId, id
+ FROM (SELECT ${fk} AS repositoryId,
+ id,
ROW_NUMBER() OVER (PARTITION BY ${fk} ORDER BY ${filesRecents.mtime} DESC) AS rn
FROM ${filesRecents}
WHERE ${fk} IS NOT NULL) AS ranked