The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
import { type Token, type Tokens, marked } from 'marked';
const BLOCK_TOKEN_TYPES = new Set([
'paragraph',
'heading',
'blockquote',
'list_item',
]);
const SKIPPED_TOKEN_TYPES = new Set(['code', 'html', 'hr', 'space', 'br']);
const getTokensText = (tokens: Token[]): string =>
tokens.map(getTokenText).join('');
const getTableCellsText = (cells: Tokens.TableCell[]): string =>
cells.map((cell) => `${getTokensText(cell.tokens)} `).join('');
const getTokenText = (token: Token): string => {
if (SKIPPED_TOKEN_TYPES.has(token.type)) {
return ' ';
}
if (token.type === 'list' && Array.isArray(token.items)) {
return getTokensText(token.items);
}
if (token.type === 'table' && Array.isArray(token.rows)) {
return [token.header, ...token.rows].map(getTableCellsText).join('');
}
if ('tokens' in token && Array.isArray(token.tokens)) {
const text = getTokensText(token.tokens);
return BLOCK_TOKEN_TYPES.has(token.type) ? `${text} ` : text;
}
return 'text' in token ? token.text : '';
};
export const getPlainTextFromMarkdownTokens = (tokens: Token[]): string =>
getTokensText(tokens).replace(/\s+/g, ' ').trim();
export const getPlainTextFromMarkdown = (markdown: string): string =>
getPlainTextFromMarkdownTokens(marked.lexer(markdown));