Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): add indexing support for markdown files

johavenabf59e7bea19
diff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.ts
index aba74315..b688da7c 100644
--- a/backend/src/applications/files/constants/indexing.ts
+++ b/backend/src/applications/files/constants/indexing.ts
@@ -2,4 +2,4 @@ export const userIndexPrefix = 'user_'
export const spaceIndexPrefix = 'space_'
export const shareIndexPrefix = 'share_'
export const minCharsToSearch = 3
-export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'html', 'htm'])
+export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])
diff --git a/backend/src/applications/files/utils/doc-textify/adapters/markdown.ts b/backend/src/applications/files/utils/doc-textify/adapters/markdown.ts
new file mode 100644
index 00000000..9b5c6638
--- /dev/null
+++ b/backend/src/applications/files/utils/doc-textify/adapters/markdown.ts
@@ -0,0 +1,12 @@
+import fs from 'fs/promises'
+
+const regexMarkdownCommon = /[`*_#>|[\]()!~-]/g
+
+function normalizeMarkdown(content: string): string {
+ return content.replace(regexMarkdownCommon, ' ')
+}
+
+export async function parseMarkdown(filePath: string): Promise<string> {
+ const content = await fs.readFile(filePath, { encoding: 'utf8' })
+ return normalizeMarkdown(content)
+}
diff --git a/backend/src/applications/files/utils/doc-textify/doc-textify.ts b/backend/src/applications/files/utils/doc-textify/doc-textify.ts
index 1699ecf6..df65c5eb 100644
--- a/backend/src/applications/files/utils/doc-textify/doc-textify.ts
+++ b/backend/src/applications/files/utils/doc-textify/doc-textify.ts
@@ -2,6 +2,7 @@ import fs from 'node:fs/promises'
import path from 'node:path'
import { parseExcel } from './adapters/excel'
import { parseHtml } from './adapters/html'
+import { parseMarkdown } from './adapters/markdown'
import { parseOpenOffice } from './adapters/open-office'
import { parsePdf } from './adapters/pdf'
import { parsePowerPoint } from './adapters/power-point'
@@ -47,10 +48,12 @@ export async function docTextify(
return cleanContent(await parsePdf(filePath, options), options)
case 'txt':
return cleanContent(await parseText(filePath), options)
+ case 'md':
+ return cleanContent(await parseMarkdown(filePath), options)
case 'html':
case 'htm':
return cleanContent(await parseHtml(filePath), options)
default:
- throw new Error('currently only supports docx, pptx, xlsx, odt, odp, ods, pdf files')
+ throw new Error('currently only supports docx, pptx, xlsx, odt, odp, ods, pdf, txt, md, html files')
}
}