Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
/* Copyright 2014 Mozilla Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Used by _extractText in FindController
*/
// source: PDFFindController pdf.js
const CHARACTERS_TO_NORMALIZE = {
'\u2010': '-', // Hyphen
'\u2018': "'", // Left single quotation mark
'\u2019': "'", // Right single quotation mark
'\u201A': "'", // Single low-9 quotation mark
'\u201B': "'", // Single high-reversed-9 quotation mark
'\u201C': '"', // Left double quotation mark
'\u201D': '"', // Right double quotation mark
'\u201E': '"', // Double low-9 quotation mark
'\u201F': '"', // Double high-reversed-9 quotation mark
'\u00BC': '1/4', // Vulgar fraction one quarter
'\u00BD': '1/2', // Vulgar fraction one half
'\u00BE': '3/4', // Vulgar fraction three quarters
};
function isCharacterToNormalize(
value: string
): value is keyof typeof CHARACTERS_TO_NORMALIZE {
return value in CHARACTERS_TO_NORMALIZE;
}
// These diacritics aren't considered as combining diacritics
// when searching in a document:
// https://searchfox.org/mozilla-central/source/intl/unicharutil/util/is_combining_diacritic.py.
// The combining class definitions can be found:
// https://www.unicode.org/reports/tr44/#Canonical_Combining_Class_Values
// Category 0 corresponds to [^\p{Mn}].
const DIACRITICS_REG_EXP = /\p{M}+/gu;
// The range [AC00-D7AF] corresponds to the Hangul syllables.
// The few other chars are some CJK Compatibility Ideographs.
const SYLLABLES_REG_EXP = /[\uAC00-\uD7AF\uFA6C\uFACF-\uFAD1\uFAD5-\uFAD7]+/g;
const SYLLABLES_LENGTHS = new Map();
// When decomposed (in using NFD) the above syllables will start
// with one of the chars in this regexp.
const FIRST_CHAR_SYLLABLES_REG_EXP =
'[\\u1100-\\u1112\\ud7a4-\\ud7af\\ud84a\\ud84c\\ud850\\ud854\\ud857\\ud85f]';
let noSyllablesRegExp: RegExp | null = null;
let withSyllablesRegExp: RegExp | null = null;
/**
* Used by _extractText in FindController
*/
export function normalize(text: string) {
// The diacritics in the text or in the query can be composed or not.
// So we use a decomposed text using NFD (and the same for the query)
// in order to be sure that diacritics are in the same order.
// Collect syllables length and positions.
const syllablePositions: number[][] = [];
let m: any;
// biome-ignore lint/suspicious/noAssignInExpressions: exception
while ((m = SYLLABLES_REG_EXP.exec(text)) !== null) {
let { index } = m;
for (const char of m[0]) {
let len = SYLLABLES_LENGTHS.get(char);
if (!len) {
len = char.normalize('NFD').length;
SYLLABLES_LENGTHS.set(char, len);
}
syllablePositions.push([len, index++]);
}
}
let normalizationRegex: RegExp;
if (syllablePositions.length === 0 && noSyllablesRegExp) {
normalizationRegex = noSyllablesRegExp;
} else if (syllablePositions.length > 0 && withSyllablesRegExp) {
normalizationRegex = withSyllablesRegExp;
} else {
// Compile the regular expression for text normalization once.
const replace = Object.keys(CHARACTERS_TO_NORMALIZE).join('');
const regexp = `([${replace}])|(\\p{M}+(?:-\\n)?)|(\\S-\\n)|(\\n)`;
if (syllablePositions.length === 0) {
// Most of the syllables belong to Hangul so there are no need
// to search for them in a non-Hangul document.
// We use the \0 in order to have the same number of groups.
normalizationRegex = noSyllablesRegExp = new RegExp(
regexp + '|(\\u0000)',
'gum'
);
} else {
normalizationRegex = withSyllablesRegExp = new RegExp(
regexp + `|(${FIRST_CHAR_SYLLABLES_REG_EXP})`,
'gum'
);
}
}
// The goal of this function is to normalize the string and
// be able to get from an index in the new string the
// corresponding index in the old string.
// For example if we have: abCd12ef456gh where C is replaced by ccc
// and numbers replaced by nothing (it's the case for diacritics), then
// we'll obtain the normalized string: abcccdefgh.
// So here the reverse map is: [0,1,2,2,2,3,6,7,11,12].
// The goal is to obtain the array: [[0, 0], [3, -1], [4, -2],
// [6, 0], [8, 3]].
// which can be used like this:
// - let say that i is the index in new string and j the index
// the old string.
// - if i is in [0; 3[ then j = i + 0
// - if i is in [3; 4[ then j = i - 1
// - if i is in [4; 6[ then j = i - 2
// ...
// Thanks to a binary search it's easy to know where is i and what's the
// shift.
// Let say that the last entry in the array is [x, s] and we have a
// substitution at index y (old string) which will replace o chars by n chars.
// Firstly, if o === n, then no need to add a new entry: the shift is
// the same.
// Secondly, if o < n, then we push the n - o elements:
// [y - (s - 1), s - 1], [y - (s - 2), s - 2], ...
// Thirdly, if o > n, then we push the element: [y - (s - n), o + s - n]
// Collect diacritics length and positions.
const rawDiacriticsPositions: number[][] = [];
// biome-ignore lint/suspicious/noAssignInExpressions: exception
while ((m = DIACRITICS_REG_EXP.exec(text)) !== null) {
rawDiacriticsPositions.push([m[0].length, m.index]);
}
let normalized = text.normalize('NFD');
const positions = [[0, 0]];
let rawDiacriticsIndex = 0;
let syllableIndex = 0;
let shift = 0;
let shiftOrigin = 0;
let eol = 0;
let hasDiacritics = false;
normalized = normalized.replace(
normalizationRegex,
(
match: string,
p1: string | undefined,
p2: string | undefined,
p3: string | undefined,
p4: string | undefined,
p5: string | undefined,
i: number
) => {
i -= shiftOrigin;
if (p1) {
// Maybe fractions or quotations mark...
if (!isCharacterToNormalize(match)) {
throw new Error(`Unexpected normalization character: ${match}`);
}
const replacement = CHARACTERS_TO_NORMALIZE[match];
const jj = replacement.length;
for (let j = 1; j < jj; j++) {
positions.push([i - shift + j, shift - j]);
}
shift -= jj - 1;
return replacement;
}
if (p2) {
const hasTrailingDashEOL = p2.endsWith('\n');
const len = hasTrailingDashEOL ? p2.length - 2 : p2.length;
// Diacritics.
hasDiacritics = true;
let jj = len;
if (i + eol === rawDiacriticsPositions[rawDiacriticsIndex]?.[1]) {
jj -= rawDiacriticsPositions[rawDiacriticsIndex][0];
++rawDiacriticsIndex;
}
for (let j = 1; j <= jj; j++) {
// i is the position of the first diacritic
// so (i - 1) is the position for the letter before.
positions.push([i - 1 - shift + j, shift - j]);
}
shift -= jj;
shiftOrigin += jj;
if (hasTrailingDashEOL) {
// Diacritics are followed by a -\n.
// See comments in `if (p3)` block.
i += len - 1;
positions.push([i - shift + 1, 1 + shift]);
shift += 1;
shiftOrigin += 1;
eol += 1;
return p2.slice(0, len);
}
return p2;
}
if (p3) {
// "X-\n" is removed because an hyphen at the end of a line
// with not a space before is likely here to mark a break
// in a word.
// The \n isn't in the original text so here y = i, n = 1 and o = 2.
positions.push([i - shift + 1, 1 + shift]);
shift += 1;
shiftOrigin += 1;
eol += 1;
return p3.charAt(0);
}
if (p4) {
// eol is replaced by space: "foo\nbar" is likely equivalent to
// "foo bar".
positions.push([i - shift + 1, shift - 1]);
shift -= 1;
shiftOrigin += 1;
eol += 1;
return ' ';
}
// p5
if (i + eol === syllablePositions[syllableIndex]?.[1]) {
// A syllable (1 char) is replaced with several chars (n) so
// newCharsLen = n - 1.
const newCharLen = syllablePositions[syllableIndex][0] - 1;
++syllableIndex;
for (let j = 1; j <= newCharLen; j++) {
positions.push([i - (shift - j), shift - j]);
}
shift -= newCharLen;
shiftOrigin += newCharLen;
}
return p5 ?? match;
}
);
positions.push([normalized.length, shift]);
return [normalized, positions, hasDiacritics];
}