Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): add indexing support for markdown files
johavenabf59e7bea19
diff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.tsindex aba74315..b688da7c 100644--- a/backend/src/applications/files/constants/indexing.ts+++ b/backend/src/applications/files/constants/indexing.ts@@ -2,4 +2,4 @@ export const userIndexPrefix = 'user_'export const spaceIndexPrefix = 'space_'export const shareIndexPrefix = 'share_'export const minCharsToSearch = 3-export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'html', 'htm'])+export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])diff --git a/backend/src/applications/files/utils/doc-textify/adapters/markdown.ts b/backend/src/applications/files/utils/doc-textify/adapters/markdown.tsnew file mode 100644index 00000000..9b5c6638--- /dev/null+++ b/backend/src/applications/files/utils/doc-textify/adapters/markdown.ts@@ -0,0 +1,12 @@+import fs from 'fs/promises'++const regexMarkdownCommon = /[`*_#>|[\]()!~-]/g++function normalizeMarkdown(content: string): string {+ return content.replace(regexMarkdownCommon, ' ')+}++export async function parseMarkdown(filePath: string): Promise<string> {+ const content = await fs.readFile(filePath, { encoding: 'utf8' })+ return normalizeMarkdown(content)+}diff --git a/backend/src/applications/files/utils/doc-textify/doc-textify.ts b/backend/src/applications/files/utils/doc-textify/doc-textify.tsindex 1699ecf6..df65c5eb 100644--- a/backend/src/applications/files/utils/doc-textify/doc-textify.ts+++ b/backend/src/applications/files/utils/doc-textify/doc-textify.ts@@ -2,6 +2,7 @@ import fs from 'node:fs/promises'import path from 'node:path'import { parseExcel } from './adapters/excel'import { parseHtml } from './adapters/html'+import { parseMarkdown } from './adapters/markdown'import { parseOpenOffice } from './adapters/open-office'import { parsePdf } from './adapters/pdf'import { parsePowerPoint } from './adapters/power-point'@@ -47,10 +48,12 @@ export async function docTextify(return cleanContent(await parsePdf(filePath, options), options)case 'txt':return cleanContent(await parseText(filePath), options)+ case 'md':+ return cleanContent(await parseMarkdown(filePath), options)case 'html':case 'htm':return cleanContent(await parseHtml(filePath), options)default:- throw new Error('currently only supports docx, pptx, xlsx, odt, odp, ods, pdf files')+ throw new Error('currently only supports docx, pptx, xlsx, odt, odp, ods, pdf, txt, md, html files')}}