A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
perf(api): compile the phrase blocklist into one matcher (#2164)
Hampusd43d242b1640
diff --git a/fluxer_api/src/api/Validator.ts b/fluxer_api/src/api/Validator.tsindex 44ef94e83..276677a52 100644--- a/fluxer_api/src/api/Validator.ts+++ b/fluxer_api/src/api/Validator.ts@@ -10,7 +10,7 @@ import type {ValidationError} from '@fluxer/errors/src/domains/core/ValidationErimport type {Context, Env, Input, MiddlewareHandler, TypedResponse, ValidationTargets} from 'hono';import {getCookie} from 'hono/cookie';import type {ZodError, ZodTypeAny} from 'zod';-import {parseJsonPreservingLargeIntegers} from './utils/LosslessJsonParser';+import {readRequestJsonBody} from './utils/RequestJsonBody';import {initializeFluxerErrorMap} from './ZodErrorMap';initializeFluxerErrorMap();@@ -200,12 +200,7 @@ export const Validator = <let value: unknown;switch (target) {case 'json':- try {- const raw = await c.req.text();- value = raw.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(raw);- } catch {- value = {};- }+ value = (await readRequestJsonBody(c.req)).value;break;case 'form': {const formData = await c.req.formData();diff --git a/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts b/fluxer_api/src/api/middleware/ContentFilterMiddleware.tsindex 634b05c66..edbb59667 100644--- a/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts+++ b/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts@@ -3,6 +3,7 @@import {ContentBlockedError} from '@fluxer/errors/src/domains/content/ContentBlockedError';import {createMiddleware} from 'hono/factory';import {Logger} from '../Logger';+import {readRequestJsonBody} from '../utils/RequestJsonBody';import {extractUrlCandidates} from '../utils/UrlNormalizer';import {phraseBlocklistCache} from './PhraseBlocklistCache';import {urlBlocklistCache} from './UrlBlocklistCache';@@ -136,13 +137,11 @@ const ContentFilterMiddleware = createMiddleware(async (ctx, next) => {if (!contentType.includes('application/json')) {return next();}- let body: unknown;- try {- body = await ctx.req.json();- } catch {+ const body = await readRequestJsonBody(ctx.req);+ if (!body.parsed) {return next();}- const strings = extractStringValues(body);+ const strings = extractStringValues(body.value);if (strings.length === 0) {return next();}diff --git a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts b/fluxer_api/src/api/middleware/PhraseBlocklistCache.tsindex 76c0ab76d..a53c6dcad 100644--- a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts+++ b/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts@@ -5,14 +5,16 @@ import {AdminRepository} from '../admin/AdminRepository';import {BANNED_PHRASES_REFRESH_CHANNEL} from '../constants/ContentModeration';import {Logger} from '../Logger';import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../utils/PhraseBlocklistNormalization';+import {SubstringMatcher} from '../utils/SubstringMatcher';export class PhraseBlocklistCache {private rawPhrases: Array<string> = [];private rawPhraseSet = new Set<string>();- private wordPhrases: Array<string> = [];- private compactPhrases: Array<string> = [];- private asciiWordPhrases: Array<string> = [];- private asciiCompactPhrases: Array<string> = [];+ private rawMatcher: SubstringMatcher | null = null;+ private wordMatcher: SubstringMatcher | null = null;+ private compactMatcher: SubstringMatcher | null = null;+ private asciiWordMatcher: SubstringMatcher | null = null;+ private asciiCompactMatcher: SubstringMatcher | null = null;private isInitialized = false;private adminRepository = new AdminRepository();private kvClient: IKVProvider | null = null;@@ -81,22 +83,14 @@ export class PhraseBlocklistCache {if (this.rawPhrases.length === 0) return false;const forms = buildPhraseMatchForms(text);return (- this.matchAny(forms.raw, this.rawPhrases) ||- this.matchAny(forms.words, this.wordPhrases) ||- this.matchAny(forms.compact, this.compactPhrases) ||- this.matchAny(forms.asciiWords, this.asciiWordPhrases) ||- this.matchAny(forms.asciiCompact, this.asciiCompactPhrases)+ this.rawMatcher?.test(forms.raw) === true ||+ this.wordMatcher?.test(forms.words) === true ||+ this.compactMatcher?.test(forms.compact) === true ||+ this.asciiWordMatcher?.test(forms.asciiWords) === true ||+ this.asciiCompactMatcher?.test(forms.asciiCompact) === true);}- private matchAny(text: string, phrases: Array<string>): boolean {- if (!text || phrases.length === 0) return false;- for (const phrase of phrases) {- if (text.includes(phrase)) return true;- }- return false;- }-private rebuildMatchers(): void {const rawPhraseSet = new Set<string>();const wordPhraseSet = new Set<string>();@@ -115,10 +109,11 @@ export class PhraseBlocklistCache {}this.rawPhraseSet = rawPhraseSet;this.rawPhrases = Array.from(rawPhraseSet);- this.wordPhrases = Array.from(wordPhraseSet);- this.compactPhrases = Array.from(compactPhraseSet);- this.asciiWordPhrases = Array.from(asciiWordPhraseSet);- this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet);+ this.rawMatcher = SubstringMatcher.fromPatterns(rawPhraseSet);+ this.wordMatcher = SubstringMatcher.fromPatterns(wordPhraseSet);+ this.compactMatcher = SubstringMatcher.fromPatterns(compactPhraseSet);+ this.asciiWordMatcher = SubstringMatcher.fromPatterns(asciiWordPhraseSet);+ this.asciiCompactMatcher = SubstringMatcher.fromPatterns(asciiCompactPhraseSet);}isPhraseBanned(phrase: string): boolean {diff --git a/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts b/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.tsnew file mode 100644index 000000000..a1e9bc808--- /dev/null+++ b/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts@@ -0,0 +1,498 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {transliterate} from 'transliteration';+import {describe, expect, test} from 'vitest';+import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../../utils/PhraseBlocklistNormalization';+import {SubstringMatcher} from '../../utils/SubstringMatcher';+import {PhraseBlocklistCache} from '../PhraseBlocklistCache';++const LEGACY_CONTROL_OR_FORMAT_REGEX = /[\p{Cc}\p{Cf}\uFFFE\uFFFF]/gu;+const LEGACY_VARIATION_SELECTOR_REGEX = /(?:[\uFE00-\uFE0F]|[\u{E0100}-\u{E01EF}])/gu;+const LEGACY_COMBINING_MARKS_REGEX = /\p{M}+/gu;+const LEGACY_NON_ALPHANUMERIC_REGEX = /[^\p{L}\p{N}]+/gu;+const LEGACY_ASCII_NON_ALPHANUMERIC_REGEX = /[^a-z0-9]+/g;+const LEGACY_WHITESPACE_REGEX = /\s+/gu;+const LEGACY_MIN_AGGRESSIVE_FORM_LENGTH = 3;++interface LegacyForms {+ raw: string;+ words: string;+ compact: string;+ asciiWords: string;+ asciiCompact: string;+}++function legacyCollapseWhitespace(value: string): string {+ return value.replace(LEGACY_WHITESPACE_REGEX, ' ').trim();+}++function legacyStripIgnorableCharacters(value: string): string {+ return value.replace(LEGACY_CONTROL_OR_FORMAT_REGEX, '').replace(LEGACY_VARIATION_SELECTOR_REGEX, '');+}++function legacyMaybeKeepAggressiveForm(value: string): string {+ return value.length >= LEGACY_MIN_AGGRESSIVE_FORM_LENGTH ? value : '';+}++function legacyBuildWordForm(value: string): string {+ return legacyCollapseWhitespace(+ value.normalize('NFKD').replace(LEGACY_COMBINING_MARKS_REGEX, '').replace(LEGACY_NON_ALPHANUMERIC_REGEX, ' '),+ );+}++function legacyBuildAsciiWordForm(value: string): string {+ return legacyCollapseWhitespace(+ transliterate(value)+ .toLowerCase()+ .normalize('NFKD')+ .replace(LEGACY_COMBINING_MARKS_REGEX, '')+ .replace(LEGACY_ASCII_NON_ALPHANUMERIC_REGEX, ' '),+ );+}++function legacyCanonicalize(value: string): string {+ return legacyStripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim();+}++function legacyBuildPhraseMatchForms(value: string): LegacyForms {+ const raw = legacyCanonicalize(value);+ const words = legacyMaybeKeepAggressiveForm(legacyBuildWordForm(raw));+ const compact = legacyMaybeKeepAggressiveForm(words.replace(LEGACY_WHITESPACE_REGEX, ''));+ const asciiWords = legacyMaybeKeepAggressiveForm(legacyBuildAsciiWordForm(raw));+ const asciiCompact = legacyMaybeKeepAggressiveForm(asciiWords.replace(LEGACY_WHITESPACE_REGEX, ''));+ return {raw, words, compact, asciiWords, asciiCompact};+}++class LegacyPhraseBlocklist {+ private rawPhrases: Array<string> = [];+ private rawPhraseSet = new Set<string>();+ private wordPhrases: Array<string> = [];+ private compactPhrases: Array<string> = [];+ private asciiWordPhrases: Array<string> = [];+ private asciiCompactPhrases: Array<string> = [];++ add(phrase: string): void {+ const canonical = legacyCanonicalize(phrase);+ if (!canonical || this.rawPhraseSet.has(canonical)) return;+ this.rawPhrases.push(canonical);+ this.rebuildMatchers();+ }++ remove(phrase: string): void {+ const canonical = legacyCanonicalize(phrase);+ if (!canonical || !this.rawPhraseSet.has(canonical)) return;+ this.rawPhrases = this.rawPhrases.filter((item) => item !== canonical);+ this.rebuildMatchers();+ }++ private rebuildMatchers(): void {+ const rawPhraseSet = new Set<string>();+ const wordPhraseSet = new Set<string>();+ const compactPhraseSet = new Set<string>();+ const asciiWordPhraseSet = new Set<string>();+ const asciiCompactPhraseSet = new Set<string>();+ for (const phrase of this.rawPhrases) {+ const canonical = legacyCanonicalize(phrase);+ if (!canonical) continue;+ const forms = legacyBuildPhraseMatchForms(canonical);+ rawPhraseSet.add(forms.raw);+ if (forms.words) wordPhraseSet.add(forms.words);+ if (forms.compact) compactPhraseSet.add(forms.compact);+ if (forms.asciiWords) asciiWordPhraseSet.add(forms.asciiWords);+ if (forms.asciiCompact) asciiCompactPhraseSet.add(forms.asciiCompact);+ }+ this.rawPhraseSet = rawPhraseSet;+ this.rawPhrases = Array.from(rawPhraseSet);+ this.wordPhrases = Array.from(wordPhraseSet);+ this.compactPhrases = Array.from(compactPhraseSet);+ this.asciiWordPhrases = Array.from(asciiWordPhraseSet);+ this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet);+ }++ private matchAny(text: string, phrases: Array<string>): boolean {+ if (!text || phrases.length === 0) return false;+ for (const phrase of phrases) {+ if (text.includes(phrase)) return true;+ }+ return false;+ }++ containsBannedPhrase(text: string): boolean {+ if (this.rawPhrases.length === 0) return false;+ const forms = legacyBuildPhraseMatchForms(text);+ return (+ this.matchAny(forms.raw, this.rawPhrases) ||+ this.matchAny(forms.words, this.wordPhrases) ||+ this.matchAny(forms.compact, this.compactPhrases) ||+ this.matchAny(forms.asciiWords, this.asciiWordPhrases) ||+ this.matchAny(forms.asciiCompact, this.asciiCompactPhrases)+ );+ }++ isPhraseBanned(phrase: string): boolean {+ const canonical = legacyCanonicalize(phrase);+ return !!canonical && this.rawPhraseSet.has(canonical);+ }++ get size(): number {+ return this.rawPhraseSet.size;+ }+}++function createRandom(seed: number): () => number {+ let state = seed >>> 0;+ return () => {+ state = (state + 0x6d2b79f5) >>> 0;+ let value = state;+ value = Math.imul(value ^ (value >>> 15), value | 1);+ value ^= value + Math.imul(value ^ (value >>> 7), value | 61);+ return ((value ^ (value >>> 14)) >>> 0) / 4294967296;+ };+}++const CORPUS_ALPHABET: ReadonlyArray<string> = [+ 'a',+ 'b',+ 'c',+ 'n',+ 'o',+ 'r',+ 't',+ 'u',+ '0',+ '1',+ '4',+ '8',+ ' ',+ ' ',+ '\t',+ '\n',+ '.',+ '-',+ '_',+ '/',+ '\\',+ '+',+ '*',+ '?',+ '[',+ ']',+ '(',+ ')',+ '{',+ '}',+ '|',+ '^',+ '$',+ '\u200B',+ '\u200C',+ '\u200D',+ '\uFE0F',+ '\u0301',+ '\u0335',+ '\u3000',+ '\uFF55',+ '\uFF4E',+ '\u043E',+ '\u03BF',+ '\u00E9',+ 'ß',+ 'fi',+ '中',+ 'א',+ '🔥',+ '😀',+];++function randomString(random: () => number, maxUnits: number): string {+ const count = Math.floor(random() * (maxUnits + 1));+ let result = '';+ for (let index = 0; index < count; index++) {+ result += CORPUS_ALPHABET[Math.floor(random() * CORPUS_ALPHABET.length)]!;+ }+ return result;+}++const TABLE_PHRASE_LISTS: ReadonlyArray<ReadonlyArray<string>> = [+ [],+ [''],+ ['a'],+ ['ab', 'b'],+ ['abc', 'bc', 'c'],+ ['unban tor', 'unban', 'ban'],+ ['c++', 'c+', '++'],+ ['a.b', 'a*b', 'a|b', 'a(b)c', 'a[b]c', 'a{2}', '^a$', 'a\\b', 'a?b'],+ ['🔥', '🔥🔥'],+ ['он', 'on', 'o'],+ ['\u00E9', 'e', 'e\u0301'],+ ['un', 'un'],+ ['aaa', 'aa', 'a'],+ ['aaaa', 'aab', 'aba', 'baa'],+ ['\u200B', 'x\u200By'],+ ['tor', 'rot', 'ort'],+];++const TABLE_TEXTS: ReadonlyArray<string> = [+ '',+ 'a',+ 'ab',+ 'abc',+ 'xabcx',+ 'aa',+ 'aaa',+ 'aaaa',+ 'banana',+ 'please UNBAN TOR right now',+ 'u n b a n t o r',+ 'u.n-b_a_n t/o\\r',+ 'unban tor',+ 'u\u200Bn\u200Bb\u200Ba\u200Bn t\u200Co\u200Dr',+ 'u̵n̵b̵a̵n̵ t̵o̵r̵',+ 'unban tоr',+ 'unban tοr',+ 'please keep tor banned',+ 'ship c++ code',+ 'compiler',+ 'a\\b',+ 'a?b',+ '^a$',+ 'a{2}',+ '🔥🔥🔥',+ 'x🔥y',+ 'caf\u00E9',+ 'cafe\u0301',+ '中中中',+ 'אא',+ ' ',+ '\t\n\t',+ '\u200B\u200C\u200D',+];++function collectPhraseMismatches(phrases: ReadonlyArray<string>, texts: ReadonlyArray<string>): Array<string> {+ const legacy = new LegacyPhraseBlocklist();+ const current = new PhraseBlocklistCache();+ for (const phrase of phrases) {+ legacy.add(phrase);+ current.add(phrase);+ }+ const mismatches: Array<string> = [];+ if (legacy.size !== current.size) {+ mismatches.push(`size ${JSON.stringify(phrases)}: legacy=${legacy.size} current=${current.size}`);+ }+ for (const phrase of phrases) {+ const expected = legacy.isPhraseBanned(phrase);+ const actual = current.isPhraseBanned(phrase);+ if (expected !== actual) {+ mismatches.push(`isPhraseBanned ${JSON.stringify(phrase)}: legacy=${expected} current=${actual}`);+ }+ }+ for (const text of texts) {+ const expected = legacy.containsBannedPhrase(text);+ const actual = current.containsBannedPhrase(text);+ if (expected !== actual) {+ mismatches.push(+ `containsBannedPhrase ${JSON.stringify(phrases)} / ${JSON.stringify(text)}: legacy=${expected} current=${actual}`,+ );+ }+ }+ return mismatches;+}++describe('phrase blocklist matcher equivalence', () => {+ test('table-driven phrase lists match the legacy linear scan', () => {+ const mismatches: Array<string> = [];+ for (const phrases of TABLE_PHRASE_LISTS) {+ mismatches.push(...collectPhraseMismatches(phrases, TABLE_TEXTS));+ }+ expect(mismatches).toEqual([]);+ });++ test('generated phrase lists and texts match the legacy linear scan', () => {+ const random = createRandom(0x5eed1234);+ const mismatches: Array<string> = [];+ for (let round = 0; round < 300; round++) {+ const phraseCount = 1 + Math.floor(random() * 8);+ const phrases: Array<string> = [];+ for (let index = 0; index < phraseCount; index++) {+ phrases.push(randomString(random, 6));+ }+ const texts: Array<string> = [];+ for (let index = 0; index < 8; index++) {+ texts.push(randomString(random, 40));+ }+ for (const phrase of phrases) {+ texts.push(phrase);+ texts.push(`${randomString(random, 5)}${phrase}${randomString(random, 5)}`);+ }+ mismatches.push(...collectPhraseMismatches(phrases, texts));+ }+ expect(mismatches).toEqual([]);+ });++ test('removal keeps both implementations in step', () => {+ const random = createRandom(0x1234abcd);+ const mismatches: Array<string> = [];+ const legacy = new LegacyPhraseBlocklist();+ const current = new PhraseBlocklistCache();+ const phrases: Array<string> = [];+ for (let index = 0; index < 40; index++) {+ phrases.push(randomString(random, 6));+ }+ for (const phrase of phrases) {+ legacy.add(phrase);+ current.add(phrase);+ }+ for (let index = 0; index < phrases.length; index += 3) {+ legacy.remove(phrases[index]!);+ current.remove(phrases[index]!);+ }+ for (let index = 0; index < 200; index++) {+ const text = randomString(random, 30);+ const expected = legacy.containsBannedPhrase(text);+ const actual = current.containsBannedPhrase(text);+ if (expected !== actual) {+ mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`);+ }+ }+ expect(mismatches).toEqual([]);+ });++ test('a large blocklist stays exact', () => {+ const random = createRandom(0xfeedface);+ const legacy = new LegacyPhraseBlocklist();+ const current = new PhraseBlocklistCache();+ for (let index = 0; index < 300; index++) {+ const phrase = `${randomString(random, 4)}${index}`;+ legacy.add(phrase);+ current.add(phrase);+ }+ expect(current.size).toBe(legacy.size);+ const mismatches: Array<string> = [];+ for (let index = 0; index < 300; index++) {+ const text = randomString(random, 200);+ const expected = legacy.containsBannedPhrase(text);+ const actual = current.containsBannedPhrase(text);+ if (expected !== actual) {+ mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`);+ }+ }+ expect(mismatches).toEqual([]);+ });+});++describe('buildPhraseMatchForms laziness', () => {+ test('lazy forms equal the eager forms for the whole corpus', () => {+ const random = createRandom(0xc0ffee11);+ const values: Array<string> = [...TABLE_TEXTS];+ for (let index = 0; index < 500; index++) {+ values.push(randomString(random, 40));+ }+ const mismatches: Array<string> = [];+ for (const value of values) {+ const expected = legacyBuildPhraseMatchForms(value);+ const forms = buildPhraseMatchForms(value);+ const actual = {+ raw: forms.raw,+ words: forms.words,+ compact: forms.compact,+ asciiWords: forms.asciiWords,+ asciiCompact: forms.asciiCompact,+ };+ if (JSON.stringify(expected) !== JSON.stringify(actual)) {+ mismatches.push(`${JSON.stringify(value)}: ${JSON.stringify(expected)} vs ${JSON.stringify(actual)}`);+ }+ }+ expect(mismatches).toEqual([]);+ });++ test('reading a later form after an earlier one is still correct', () => {+ const value = 'unban tоr';+ const forms = buildPhraseMatchForms(value);+ const expected = legacyBuildPhraseMatchForms(value);+ expect(forms.asciiCompact).toBe(expected.asciiCompact);+ expect(forms.raw).toBe(expected.raw);+ expect(forms.words).toBe(expected.words);+ expect(forms.compact).toBe(expected.compact);+ expect(forms.asciiWords).toBe(expected.asciiWords);+ });++ test('canonicalizeStoredPhrase is unchanged', () => {+ const random = createRandom(0x0badf00d);+ const mismatches: Array<string> = [];+ for (let index = 0; index < 500; index++) {+ const value = randomString(random, 30);+ if (canonicalizeStoredPhrase(value) !== legacyCanonicalize(value)) {+ mismatches.push(JSON.stringify(value));+ }+ }+ expect(mismatches).toEqual([]);+ });+});++describe('SubstringMatcher', () => {+ test('is null only for an empty pattern set', () => {+ expect(SubstringMatcher.fromPatterns([])).toBeNull();+ expect(SubstringMatcher.fromPatterns(new Set<string>())).toBeNull();+ expect(SubstringMatcher.fromPatterns([''])).not.toBeNull();+ });++ test('agrees with String.includes on generated patterns and texts', () => {+ const random = createRandom(0x2468ace0);+ const mismatches: Array<string> = [];+ for (let round = 0; round < 500; round++) {+ const patternCount = 1 + Math.floor(random() * 6);+ const patterns: Array<string> = [];+ for (let index = 0; index < patternCount; index++) {+ patterns.push(randomString(random, 5));+ }+ const matcher = SubstringMatcher.fromPatterns(patterns);+ if (matcher === null) {+ mismatches.push(`unexpected null for ${JSON.stringify(patterns)}`);+ continue;+ }+ for (let index = 0; index < 10; index++) {+ const text = randomString(random, 30);+ const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern));+ const actual = matcher.test(text);+ if (expected !== actual) {+ mismatches.push(`${JSON.stringify(patterns)} / ${JSON.stringify(text)}: ${expected} vs ${actual}`);+ }+ }+ }+ expect(mismatches).toEqual([]);+ });++ test('an empty pattern matches any non-empty text', () => {+ const matcher = SubstringMatcher.fromPatterns(['', 'zzz']);+ expect(matcher).not.toBeNull();+ expect(matcher!.test('')).toBe(false);+ expect(matcher!.test('anything')).toBe(true);+ });++ test('finds patterns that are suffixes of other pattern prefixes', () => {+ const matcher = SubstringMatcher.fromPatterns(['abcd', 'bc']);+ expect(matcher!.test('xabcx')).toBe(true);+ expect(matcher!.test('abd')).toBe(false);+ expect(matcher!.test('zabcdz')).toBe(true);+ });++ test('scales to a very large pattern set without a compile limit', () => {+ const random = createRandom(0x13579bdf);+ const patterns: Array<string> = [];+ for (let index = 0; index < 20000; index++) {+ patterns.push(`${randomString(random, 4)}${index}`);+ }+ const matcher = SubstringMatcher.fromPatterns(patterns);+ expect(matcher).not.toBeNull();+ const mismatches: Array<string> = [];+ for (let index = 0; index < 100; index++) {+ const text = index % 2 === 0 ? randomString(random, 400) : `${randomString(random, 50)}${patterns[index]!}`;+ const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern));+ const actual = matcher!.test(text);+ if (expected !== actual) {+ mismatches.push(`${JSON.stringify(text)}: ${expected} vs ${actual}`);+ }+ }+ expect(mismatches).toEqual([]);+ });+});diff --git a/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts b/fluxer_api/src/api/utils/PhraseBlocklistNormalization.tsindex b6b66fccf..f267c953d 100644--- a/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts+++ b/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts@@ -11,11 +11,11 @@ const WHITESPACE_REGEX = /\s+/gu;const MIN_AGGRESSIVE_FORM_LENGTH = 3;interface PhraseMatchForms {- raw: string;- words: string;- compact: string;- asciiWords: string;- asciiCompact: string;+ readonly raw: string;+ readonly words: string;+ readonly compact: string;+ readonly asciiWords: string;+ readonly asciiCompact: string;}function collapseWhitespace(value: string): string {@@ -46,21 +46,50 @@ function buildAsciiWordFormFromCanonical(value: string): string {);}+class LazyPhraseMatchForms implements PhraseMatchForms {+ readonly raw: string;+ private wordsForm: string | null = null;+ private compactForm: string | null = null;+ private asciiWordsForm: string | null = null;+ private asciiCompactForm: string | null = null;++ constructor(value: string) {+ this.raw = canonicalizeStoredPhrase(value);+ }++ get words(): string {+ if (this.wordsForm === null) {+ this.wordsForm = maybeKeepAggressiveForm(buildWordFormFromCanonical(this.raw));+ }+ return this.wordsForm;+ }++ get compact(): string {+ if (this.compactForm === null) {+ this.compactForm = maybeKeepAggressiveForm(this.words.replace(WHITESPACE_REGEX, ''));+ }+ return this.compactForm;+ }++ get asciiWords(): string {+ if (this.asciiWordsForm === null) {+ this.asciiWordsForm = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(this.raw));+ }+ return this.asciiWordsForm;+ }++ get asciiCompact(): string {+ if (this.asciiCompactForm === null) {+ this.asciiCompactForm = maybeKeepAggressiveForm(this.asciiWords.replace(WHITESPACE_REGEX, ''));+ }+ return this.asciiCompactForm;+ }+}+export function canonicalizeStoredPhrase(value: string): string {return stripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim();}export function buildPhraseMatchForms(value: string): PhraseMatchForms {- const raw = canonicalizeStoredPhrase(value);- const words = maybeKeepAggressiveForm(buildWordFormFromCanonical(raw));- const compact = maybeKeepAggressiveForm(words.replace(WHITESPACE_REGEX, ''));- const asciiWords = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(raw));- const asciiCompact = maybeKeepAggressiveForm(asciiWords.replace(WHITESPACE_REGEX, ''));- return {- raw,- words,- compact,- asciiWords,- asciiCompact,- };+ return new LazyPhraseMatchForms(value);}diff --git a/fluxer_api/src/api/utils/RequestJsonBody.ts b/fluxer_api/src/api/utils/RequestJsonBody.tsnew file mode 100644index 000000000..e9b03774a--- /dev/null+++ b/fluxer_api/src/api/utils/RequestJsonBody.ts@@ -0,0 +1,26 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import type {HonoRequest} from 'hono';+import {parseJsonPreservingLargeIntegers} from './LosslessJsonParser';++interface RequestJsonBody {+ parsed: boolean;+ value: unknown;+}++const bodyCache = new WeakMap<Request, RequestJsonBody>();++export async function readRequestJsonBody(req: HonoRequest): Promise<RequestJsonBody> {+ const raw = req.raw;+ const cached = bodyCache.get(raw);+ if (cached) return cached;+ let body: RequestJsonBody;+ try {+ const text = await req.text();+ body = {parsed: true, value: text.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(text)};+ } catch {+ body = {parsed: false, value: {}};+ }+ bodyCache.set(raw, body);+ return body;+}diff --git a/fluxer_api/src/api/utils/SubstringMatcher.ts b/fluxer_api/src/api/utils/SubstringMatcher.tsnew file mode 100644index 000000000..22083297a--- /dev/null+++ b/fluxer_api/src/api/utils/SubstringMatcher.ts@@ -0,0 +1,198 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++const ROOT_NODE = 0;+const ASCII_ROOT_SIZE = 128;+const NO_TRANSITION = -1;+const EDGE_KEY_STRIDE = 65536;++export class SubstringMatcher {+ private readonly rowStart: Int32Array;+ private readonly edgeCodes: Int32Array;+ private readonly edgeTargets: Int32Array;+ private readonly failures: Int32Array;+ private readonly terminals: Uint8Array;+ private readonly asciiRoot: Int32Array;+ private readonly matchesEverything: boolean;++ private constructor(+ rowStart: Int32Array,+ edgeCodes: Int32Array,+ edgeTargets: Int32Array,+ failures: Int32Array,+ terminals: Uint8Array,+ matchesEverything: boolean,+ ) {+ this.rowStart = rowStart;+ this.edgeCodes = edgeCodes;+ this.edgeTargets = edgeTargets;+ this.failures = failures;+ this.terminals = terminals;+ this.matchesEverything = matchesEverything;+ this.asciiRoot = new Int32Array(ASCII_ROOT_SIZE);+ for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) {+ const code = edgeCodes[edge]!;+ if (code < ASCII_ROOT_SIZE) {+ this.asciiRoot[code] = edgeTargets[edge]!;+ }+ }+ }++ static fromPatterns(patterns: Iterable<string>): SubstringMatcher | null {+ const children = new Map<number, number>();+ const edgeSources: Array<number> = [];+ const edgeLabels: Array<number> = [];+ const edgeChildren: Array<number> = [];+ const terminalNodes: Array<number> = [];+ let nodeCount = 1;+ let matchesEverything = false;+ let count = 0;+ for (const pattern of patterns) {+ count++;+ if (pattern.length === 0) {+ matchesEverything = true;+ continue;+ }+ let node = ROOT_NODE;+ for (let index = 0; index < pattern.length; index++) {+ const code = pattern.charCodeAt(index);+ const key = node * EDGE_KEY_STRIDE + code;+ let next = children.get(key);+ if (next === undefined) {+ next = nodeCount++;+ children.set(key, next);+ edgeSources.push(node);+ edgeLabels.push(code);+ edgeChildren.push(next);+ }+ node = next;+ }+ terminalNodes.push(node);+ }+ if (count === 0) {+ return null;+ }+ children.clear();+ const edgeCount = edgeSources.length;+ const rowStart = new Int32Array(nodeCount + 1);+ for (let edge = 0; edge < edgeCount; edge++) {+ rowStart[edgeSources[edge]! + 1]!++;+ }+ for (let node = 0; node < nodeCount; node++) {+ rowStart[node + 1]! += rowStart[node]!;+ }+ const cursor = rowStart.slice(0, nodeCount);+ const edgeCodes = new Int32Array(edgeCount);+ const edgeTargets = new Int32Array(edgeCount);+ for (let edge = 0; edge < edgeCount; edge++) {+ const source = edgeSources[edge]!;+ const slot = cursor[source]!;+ cursor[source] = slot + 1;+ edgeCodes[slot] = edgeLabels[edge]!;+ edgeTargets[slot] = edgeChildren[edge]!;+ }+ for (let node = 0; node < nodeCount; node++) {+ const start = rowStart[node]!;+ const end = rowStart[node + 1]!;+ for (let index = start + 1; index < end; index++) {+ const code = edgeCodes[index]!;+ const target = edgeTargets[index]!;+ let scan = index - 1;+ while (scan >= start && edgeCodes[scan]! > code) {+ edgeCodes[scan + 1] = edgeCodes[scan]!;+ edgeTargets[scan + 1] = edgeTargets[scan]!;+ scan--;+ }+ edgeCodes[scan + 1] = code;+ edgeTargets[scan + 1] = target;+ }+ }+ const failures = new Int32Array(nodeCount);+ const terminals = new Uint8Array(nodeCount);+ for (const node of terminalNodes) {+ terminals[node] = 1;+ }+ const queue = new Int32Array(nodeCount);+ let tail = 0;+ for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) {+ queue[tail] = edgeTargets[edge]!;+ tail++;+ }+ for (let head = 0; head < tail; head++) {+ const node = queue[head]!;+ if (terminals[failures[node]!]! === 1) {+ terminals[node] = 1;+ }+ for (let edge = rowStart[node]!; edge < rowStart[node + 1]!; edge++) {+ const code = edgeCodes[edge]!;+ const child = edgeTargets[edge]!;+ let candidate = failures[node]!;+ let target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code);+ while (target === NO_TRANSITION && candidate !== ROOT_NODE) {+ candidate = failures[candidate]!;+ target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code);+ }+ failures[child] = target === NO_TRANSITION ? ROOT_NODE : target;+ queue[tail] = child;+ tail++;+ }+ }+ return new SubstringMatcher(rowStart, edgeCodes, edgeTargets, failures, terminals, matchesEverything);+ }++ test(text: string): boolean {+ if (text.length === 0) {+ return false;+ }+ if (this.matchesEverything) {+ return true;+ }+ const rowStart = this.rowStart;+ const edgeCodes = this.edgeCodes;+ const edgeTargets = this.edgeTargets;+ const failures = this.failures;+ const terminals = this.terminals;+ const asciiRoot = this.asciiRoot;+ let node = ROOT_NODE;+ for (let index = 0; index < text.length; index++) {+ const code = text.charCodeAt(index);+ if (node === ROOT_NODE && code < ASCII_ROOT_SIZE) {+ node = asciiRoot[code]!;+ } else {+ let next = findTransition(rowStart, edgeCodes, edgeTargets, node, code);+ while (next === NO_TRANSITION && node !== ROOT_NODE) {+ node = failures[node]!;+ next = findTransition(rowStart, edgeCodes, edgeTargets, node, code);+ }+ node = next === NO_TRANSITION ? ROOT_NODE : next;+ }+ if (terminals[node]! === 1) {+ return true;+ }+ }+ return false;+ }+}++function findTransition(+ rowStart: Int32Array,+ edgeCodes: Int32Array,+ edgeTargets: Int32Array,+ node: number,+ code: number,+): number {+ let low = rowStart[node]!;+ let high = rowStart[node + 1]! - 1;+ while (low <= high) {+ const middle = (low + high) >>> 1;+ const candidate = edgeCodes[middle]!;+ if (candidate === code) {+ return edgeTargets[middle]!;+ }+ if (candidate < code) {+ low = middle + 1;+ } else {+ high = middle - 1;+ }+ }+ return NO_TRANSITION;+}