Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(api): compile the phrase blocklist into one matcher (#2164)

Hampusd43d242b1640
diff --git a/fluxer_api/src/api/Validator.ts b/fluxer_api/src/api/Validator.ts
index 44ef94e83..276677a52 100644
--- a/fluxer_api/src/api/Validator.ts
+++ b/fluxer_api/src/api/Validator.ts
@@ -10,7 +10,7 @@ import type {ValidationError} from '@fluxer/errors/src/domains/core/ValidationEr
import type {Context, Env, Input, MiddlewareHandler, TypedResponse, ValidationTargets} from 'hono';
import {getCookie} from 'hono/cookie';
import type {ZodError, ZodTypeAny} from 'zod';
-import {parseJsonPreservingLargeIntegers} from './utils/LosslessJsonParser';
+import {readRequestJsonBody} from './utils/RequestJsonBody';
import {initializeFluxerErrorMap} from './ZodErrorMap';
initializeFluxerErrorMap();
@@ -200,12 +200,7 @@ export const Validator = <
let value: unknown;
switch (target) {
case 'json':
- try {
- const raw = await c.req.text();
- value = raw.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(raw);
- } catch {
- value = {};
- }
+ value = (await readRequestJsonBody(c.req)).value;
break;
case 'form': {
const formData = await c.req.formData();
diff --git a/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts b/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts
index 634b05c66..edbb59667 100644
--- a/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts
+++ b/fluxer_api/src/api/middleware/ContentFilterMiddleware.ts
@@ -3,6 +3,7 @@
import {ContentBlockedError} from '@fluxer/errors/src/domains/content/ContentBlockedError';
import {createMiddleware} from 'hono/factory';
import {Logger} from '../Logger';
+import {readRequestJsonBody} from '../utils/RequestJsonBody';
import {extractUrlCandidates} from '../utils/UrlNormalizer';
import {phraseBlocklistCache} from './PhraseBlocklistCache';
import {urlBlocklistCache} from './UrlBlocklistCache';
@@ -136,13 +137,11 @@ const ContentFilterMiddleware = createMiddleware(async (ctx, next) => {
if (!contentType.includes('application/json')) {
return next();
}
- let body: unknown;
- try {
- body = await ctx.req.json();
- } catch {
+ const body = await readRequestJsonBody(ctx.req);
+ if (!body.parsed) {
return next();
}
- const strings = extractStringValues(body);
+ const strings = extractStringValues(body.value);
if (strings.length === 0) {
return next();
}
diff --git a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts b/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts
index 76c0ab76d..a53c6dcad 100644
--- a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts
+++ b/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts
@@ -5,14 +5,16 @@ import {AdminRepository} from '../admin/AdminRepository';
import {BANNED_PHRASES_REFRESH_CHANNEL} from '../constants/ContentModeration';
import {Logger} from '../Logger';
import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../utils/PhraseBlocklistNormalization';
+import {SubstringMatcher} from '../utils/SubstringMatcher';
export class PhraseBlocklistCache {
private rawPhrases: Array<string> = [];
private rawPhraseSet = new Set<string>();
- private wordPhrases: Array<string> = [];
- private compactPhrases: Array<string> = [];
- private asciiWordPhrases: Array<string> = [];
- private asciiCompactPhrases: Array<string> = [];
+ private rawMatcher: SubstringMatcher | null = null;
+ private wordMatcher: SubstringMatcher | null = null;
+ private compactMatcher: SubstringMatcher | null = null;
+ private asciiWordMatcher: SubstringMatcher | null = null;
+ private asciiCompactMatcher: SubstringMatcher | null = null;
private isInitialized = false;
private adminRepository = new AdminRepository();
private kvClient: IKVProvider | null = null;
@@ -81,22 +83,14 @@ export class PhraseBlocklistCache {
if (this.rawPhrases.length === 0) return false;
const forms = buildPhraseMatchForms(text);
return (
- this.matchAny(forms.raw, this.rawPhrases) ||
- this.matchAny(forms.words, this.wordPhrases) ||
- this.matchAny(forms.compact, this.compactPhrases) ||
- this.matchAny(forms.asciiWords, this.asciiWordPhrases) ||
- this.matchAny(forms.asciiCompact, this.asciiCompactPhrases)
+ this.rawMatcher?.test(forms.raw) === true ||
+ this.wordMatcher?.test(forms.words) === true ||
+ this.compactMatcher?.test(forms.compact) === true ||
+ this.asciiWordMatcher?.test(forms.asciiWords) === true ||
+ this.asciiCompactMatcher?.test(forms.asciiCompact) === true
);
}
- private matchAny(text: string, phrases: Array<string>): boolean {
- if (!text || phrases.length === 0) return false;
- for (const phrase of phrases) {
- if (text.includes(phrase)) return true;
- }
- return false;
- }
-
private rebuildMatchers(): void {
const rawPhraseSet = new Set<string>();
const wordPhraseSet = new Set<string>();
@@ -115,10 +109,11 @@ export class PhraseBlocklistCache {
}
this.rawPhraseSet = rawPhraseSet;
this.rawPhrases = Array.from(rawPhraseSet);
- this.wordPhrases = Array.from(wordPhraseSet);
- this.compactPhrases = Array.from(compactPhraseSet);
- this.asciiWordPhrases = Array.from(asciiWordPhraseSet);
- this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet);
+ this.rawMatcher = SubstringMatcher.fromPatterns(rawPhraseSet);
+ this.wordMatcher = SubstringMatcher.fromPatterns(wordPhraseSet);
+ this.compactMatcher = SubstringMatcher.fromPatterns(compactPhraseSet);
+ this.asciiWordMatcher = SubstringMatcher.fromPatterns(asciiWordPhraseSet);
+ this.asciiCompactMatcher = SubstringMatcher.fromPatterns(asciiCompactPhraseSet);
}
isPhraseBanned(phrase: string): boolean {
diff --git a/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts b/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts
new file mode 100644
index 000000000..a1e9bc808
--- /dev/null
+++ b/fluxer_api/src/api/middleware/tests/PhraseBlocklistEquivalence.test.ts
@@ -0,0 +1,498 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {transliterate} from 'transliteration';
+import {describe, expect, test} from 'vitest';
+import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../../utils/PhraseBlocklistNormalization';
+import {SubstringMatcher} from '../../utils/SubstringMatcher';
+import {PhraseBlocklistCache} from '../PhraseBlocklistCache';
+
+const LEGACY_CONTROL_OR_FORMAT_REGEX = /[\p{Cc}\p{Cf}\uFFFE\uFFFF]/gu;
+const LEGACY_VARIATION_SELECTOR_REGEX = /(?:[\uFE00-\uFE0F]|[\u{E0100}-\u{E01EF}])/gu;
+const LEGACY_COMBINING_MARKS_REGEX = /\p{M}+/gu;
+const LEGACY_NON_ALPHANUMERIC_REGEX = /[^\p{L}\p{N}]+/gu;
+const LEGACY_ASCII_NON_ALPHANUMERIC_REGEX = /[^a-z0-9]+/g;
+const LEGACY_WHITESPACE_REGEX = /\s+/gu;
+const LEGACY_MIN_AGGRESSIVE_FORM_LENGTH = 3;
+
+interface LegacyForms {
+ raw: string;
+ words: string;
+ compact: string;
+ asciiWords: string;
+ asciiCompact: string;
+}
+
+function legacyCollapseWhitespace(value: string): string {
+ return value.replace(LEGACY_WHITESPACE_REGEX, ' ').trim();
+}
+
+function legacyStripIgnorableCharacters(value: string): string {
+ return value.replace(LEGACY_CONTROL_OR_FORMAT_REGEX, '').replace(LEGACY_VARIATION_SELECTOR_REGEX, '');
+}
+
+function legacyMaybeKeepAggressiveForm(value: string): string {
+ return value.length >= LEGACY_MIN_AGGRESSIVE_FORM_LENGTH ? value : '';
+}
+
+function legacyBuildWordForm(value: string): string {
+ return legacyCollapseWhitespace(
+ value.normalize('NFKD').replace(LEGACY_COMBINING_MARKS_REGEX, '').replace(LEGACY_NON_ALPHANUMERIC_REGEX, ' '),
+ );
+}
+
+function legacyBuildAsciiWordForm(value: string): string {
+ return legacyCollapseWhitespace(
+ transliterate(value)
+ .toLowerCase()
+ .normalize('NFKD')
+ .replace(LEGACY_COMBINING_MARKS_REGEX, '')
+ .replace(LEGACY_ASCII_NON_ALPHANUMERIC_REGEX, ' '),
+ );
+}
+
+function legacyCanonicalize(value: string): string {
+ return legacyStripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim();
+}
+
+function legacyBuildPhraseMatchForms(value: string): LegacyForms {
+ const raw = legacyCanonicalize(value);
+ const words = legacyMaybeKeepAggressiveForm(legacyBuildWordForm(raw));
+ const compact = legacyMaybeKeepAggressiveForm(words.replace(LEGACY_WHITESPACE_REGEX, ''));
+ const asciiWords = legacyMaybeKeepAggressiveForm(legacyBuildAsciiWordForm(raw));
+ const asciiCompact = legacyMaybeKeepAggressiveForm(asciiWords.replace(LEGACY_WHITESPACE_REGEX, ''));
+ return {raw, words, compact, asciiWords, asciiCompact};
+}
+
+class LegacyPhraseBlocklist {
+ private rawPhrases: Array<string> = [];
+ private rawPhraseSet = new Set<string>();
+ private wordPhrases: Array<string> = [];
+ private compactPhrases: Array<string> = [];
+ private asciiWordPhrases: Array<string> = [];
+ private asciiCompactPhrases: Array<string> = [];
+
+ add(phrase: string): void {
+ const canonical = legacyCanonicalize(phrase);
+ if (!canonical || this.rawPhraseSet.has(canonical)) return;
+ this.rawPhrases.push(canonical);
+ this.rebuildMatchers();
+ }
+
+ remove(phrase: string): void {
+ const canonical = legacyCanonicalize(phrase);
+ if (!canonical || !this.rawPhraseSet.has(canonical)) return;
+ this.rawPhrases = this.rawPhrases.filter((item) => item !== canonical);
+ this.rebuildMatchers();
+ }
+
+ private rebuildMatchers(): void {
+ const rawPhraseSet = new Set<string>();
+ const wordPhraseSet = new Set<string>();
+ const compactPhraseSet = new Set<string>();
+ const asciiWordPhraseSet = new Set<string>();
+ const asciiCompactPhraseSet = new Set<string>();
+ for (const phrase of this.rawPhrases) {
+ const canonical = legacyCanonicalize(phrase);
+ if (!canonical) continue;
+ const forms = legacyBuildPhraseMatchForms(canonical);
+ rawPhraseSet.add(forms.raw);
+ if (forms.words) wordPhraseSet.add(forms.words);
+ if (forms.compact) compactPhraseSet.add(forms.compact);
+ if (forms.asciiWords) asciiWordPhraseSet.add(forms.asciiWords);
+ if (forms.asciiCompact) asciiCompactPhraseSet.add(forms.asciiCompact);
+ }
+ this.rawPhraseSet = rawPhraseSet;
+ this.rawPhrases = Array.from(rawPhraseSet);
+ this.wordPhrases = Array.from(wordPhraseSet);
+ this.compactPhrases = Array.from(compactPhraseSet);
+ this.asciiWordPhrases = Array.from(asciiWordPhraseSet);
+ this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet);
+ }
+
+ private matchAny(text: string, phrases: Array<string>): boolean {
+ if (!text || phrases.length === 0) return false;
+ for (const phrase of phrases) {
+ if (text.includes(phrase)) return true;
+ }
+ return false;
+ }
+
+ containsBannedPhrase(text: string): boolean {
+ if (this.rawPhrases.length === 0) return false;
+ const forms = legacyBuildPhraseMatchForms(text);
+ return (
+ this.matchAny(forms.raw, this.rawPhrases) ||
+ this.matchAny(forms.words, this.wordPhrases) ||
+ this.matchAny(forms.compact, this.compactPhrases) ||
+ this.matchAny(forms.asciiWords, this.asciiWordPhrases) ||
+ this.matchAny(forms.asciiCompact, this.asciiCompactPhrases)
+ );
+ }
+
+ isPhraseBanned(phrase: string): boolean {
+ const canonical = legacyCanonicalize(phrase);
+ return !!canonical && this.rawPhraseSet.has(canonical);
+ }
+
+ get size(): number {
+ return this.rawPhraseSet.size;
+ }
+}
+
+function createRandom(seed: number): () => number {
+ let state = seed >>> 0;
+ return () => {
+ state = (state + 0x6d2b79f5) >>> 0;
+ let value = state;
+ value = Math.imul(value ^ (value >>> 15), value | 1);
+ value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
+ return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
+ };
+}
+
+const CORPUS_ALPHABET: ReadonlyArray<string> = [
+ 'a',
+ 'b',
+ 'c',
+ 'n',
+ 'o',
+ 'r',
+ 't',
+ 'u',
+ '0',
+ '1',
+ '4',
+ '8',
+ ' ',
+ ' ',
+ '\t',
+ '\n',
+ '.',
+ '-',
+ '_',
+ '/',
+ '\\',
+ '+',
+ '*',
+ '?',
+ '[',
+ ']',
+ '(',
+ ')',
+ '{',
+ '}',
+ '|',
+ '^',
+ '$',
+ '\u200B',
+ '\u200C',
+ '\u200D',
+ '\uFE0F',
+ '\u0301',
+ '\u0335',
+ '\u3000',
+ '\uFF55',
+ '\uFF4E',
+ '\u043E',
+ '\u03BF',
+ '\u00E9',
+ 'ß',
+ 'fi',
+ '中',
+ 'א',
+ '🔥',
+ '😀',
+];
+
+function randomString(random: () => number, maxUnits: number): string {
+ const count = Math.floor(random() * (maxUnits + 1));
+ let result = '';
+ for (let index = 0; index < count; index++) {
+ result += CORPUS_ALPHABET[Math.floor(random() * CORPUS_ALPHABET.length)]!;
+ }
+ return result;
+}
+
+const TABLE_PHRASE_LISTS: ReadonlyArray<ReadonlyArray<string>> = [
+ [],
+ [''],
+ ['a'],
+ ['ab', 'b'],
+ ['abc', 'bc', 'c'],
+ ['unban tor', 'unban', 'ban'],
+ ['c++', 'c+', '++'],
+ ['a.b', 'a*b', 'a|b', 'a(b)c', 'a[b]c', 'a{2}', '^a$', 'a\\b', 'a?b'],
+ ['🔥', '🔥🔥'],
+ ['он', 'on', 'o'],
+ ['\u00E9', 'e', 'e\u0301'],
+ ['un', 'un'],
+ ['aaa', 'aa', 'a'],
+ ['aaaa', 'aab', 'aba', 'baa'],
+ ['\u200B', 'x\u200By'],
+ ['tor', 'rot', 'ort'],
+];
+
+const TABLE_TEXTS: ReadonlyArray<string> = [
+ '',
+ 'a',
+ 'ab',
+ 'abc',
+ 'xabcx',
+ 'aa',
+ 'aaa',
+ 'aaaa',
+ 'banana',
+ 'please UNBAN TOR right now',
+ 'u n b a n t o r',
+ 'u.n-b_a_n t/o\\r',
+ 'unban tor',
+ 'u\u200Bn\u200Bb\u200Ba\u200Bn t\u200Co\u200Dr',
+ 'u̵n̵b̵a̵n̵ t̵o̵r̵',
+ 'unban tоr',
+ 'unban tοr',
+ 'please keep tor banned',
+ 'ship c++ code',
+ 'compiler',
+ 'a\\b',
+ 'a?b',
+ '^a$',
+ 'a{2}',
+ '🔥🔥🔥',
+ 'x🔥y',
+ 'caf\u00E9',
+ 'cafe\u0301',
+ '中中中',
+ 'אא',
+ ' ',
+ '\t\n\t',
+ '\u200B\u200C\u200D',
+];
+
+function collectPhraseMismatches(phrases: ReadonlyArray<string>, texts: ReadonlyArray<string>): Array<string> {
+ const legacy = new LegacyPhraseBlocklist();
+ const current = new PhraseBlocklistCache();
+ for (const phrase of phrases) {
+ legacy.add(phrase);
+ current.add(phrase);
+ }
+ const mismatches: Array<string> = [];
+ if (legacy.size !== current.size) {
+ mismatches.push(`size ${JSON.stringify(phrases)}: legacy=${legacy.size} current=${current.size}`);
+ }
+ for (const phrase of phrases) {
+ const expected = legacy.isPhraseBanned(phrase);
+ const actual = current.isPhraseBanned(phrase);
+ if (expected !== actual) {
+ mismatches.push(`isPhraseBanned ${JSON.stringify(phrase)}: legacy=${expected} current=${actual}`);
+ }
+ }
+ for (const text of texts) {
+ const expected = legacy.containsBannedPhrase(text);
+ const actual = current.containsBannedPhrase(text);
+ if (expected !== actual) {
+ mismatches.push(
+ `containsBannedPhrase ${JSON.stringify(phrases)} / ${JSON.stringify(text)}: legacy=${expected} current=${actual}`,
+ );
+ }
+ }
+ return mismatches;
+}
+
+describe('phrase blocklist matcher equivalence', () => {
+ test('table-driven phrase lists match the legacy linear scan', () => {
+ const mismatches: Array<string> = [];
+ for (const phrases of TABLE_PHRASE_LISTS) {
+ mismatches.push(...collectPhraseMismatches(phrases, TABLE_TEXTS));
+ }
+ expect(mismatches).toEqual([]);
+ });
+
+ test('generated phrase lists and texts match the legacy linear scan', () => {
+ const random = createRandom(0x5eed1234);
+ const mismatches: Array<string> = [];
+ for (let round = 0; round < 300; round++) {
+ const phraseCount = 1 + Math.floor(random() * 8);
+ const phrases: Array<string> = [];
+ for (let index = 0; index < phraseCount; index++) {
+ phrases.push(randomString(random, 6));
+ }
+ const texts: Array<string> = [];
+ for (let index = 0; index < 8; index++) {
+ texts.push(randomString(random, 40));
+ }
+ for (const phrase of phrases) {
+ texts.push(phrase);
+ texts.push(`${randomString(random, 5)}${phrase}${randomString(random, 5)}`);
+ }
+ mismatches.push(...collectPhraseMismatches(phrases, texts));
+ }
+ expect(mismatches).toEqual([]);
+ });
+
+ test('removal keeps both implementations in step', () => {
+ const random = createRandom(0x1234abcd);
+ const mismatches: Array<string> = [];
+ const legacy = new LegacyPhraseBlocklist();
+ const current = new PhraseBlocklistCache();
+ const phrases: Array<string> = [];
+ for (let index = 0; index < 40; index++) {
+ phrases.push(randomString(random, 6));
+ }
+ for (const phrase of phrases) {
+ legacy.add(phrase);
+ current.add(phrase);
+ }
+ for (let index = 0; index < phrases.length; index += 3) {
+ legacy.remove(phrases[index]!);
+ current.remove(phrases[index]!);
+ }
+ for (let index = 0; index < 200; index++) {
+ const text = randomString(random, 30);
+ const expected = legacy.containsBannedPhrase(text);
+ const actual = current.containsBannedPhrase(text);
+ if (expected !== actual) {
+ mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`);
+ }
+ }
+ expect(mismatches).toEqual([]);
+ });
+
+ test('a large blocklist stays exact', () => {
+ const random = createRandom(0xfeedface);
+ const legacy = new LegacyPhraseBlocklist();
+ const current = new PhraseBlocklistCache();
+ for (let index = 0; index < 300; index++) {
+ const phrase = `${randomString(random, 4)}${index}`;
+ legacy.add(phrase);
+ current.add(phrase);
+ }
+ expect(current.size).toBe(legacy.size);
+ const mismatches: Array<string> = [];
+ for (let index = 0; index < 300; index++) {
+ const text = randomString(random, 200);
+ const expected = legacy.containsBannedPhrase(text);
+ const actual = current.containsBannedPhrase(text);
+ if (expected !== actual) {
+ mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`);
+ }
+ }
+ expect(mismatches).toEqual([]);
+ });
+});
+
+describe('buildPhraseMatchForms laziness', () => {
+ test('lazy forms equal the eager forms for the whole corpus', () => {
+ const random = createRandom(0xc0ffee11);
+ const values: Array<string> = [...TABLE_TEXTS];
+ for (let index = 0; index < 500; index++) {
+ values.push(randomString(random, 40));
+ }
+ const mismatches: Array<string> = [];
+ for (const value of values) {
+ const expected = legacyBuildPhraseMatchForms(value);
+ const forms = buildPhraseMatchForms(value);
+ const actual = {
+ raw: forms.raw,
+ words: forms.words,
+ compact: forms.compact,
+ asciiWords: forms.asciiWords,
+ asciiCompact: forms.asciiCompact,
+ };
+ if (JSON.stringify(expected) !== JSON.stringify(actual)) {
+ mismatches.push(`${JSON.stringify(value)}: ${JSON.stringify(expected)} vs ${JSON.stringify(actual)}`);
+ }
+ }
+ expect(mismatches).toEqual([]);
+ });
+
+ test('reading a later form after an earlier one is still correct', () => {
+ const value = 'unban tоr';
+ const forms = buildPhraseMatchForms(value);
+ const expected = legacyBuildPhraseMatchForms(value);
+ expect(forms.asciiCompact).toBe(expected.asciiCompact);
+ expect(forms.raw).toBe(expected.raw);
+ expect(forms.words).toBe(expected.words);
+ expect(forms.compact).toBe(expected.compact);
+ expect(forms.asciiWords).toBe(expected.asciiWords);
+ });
+
+ test('canonicalizeStoredPhrase is unchanged', () => {
+ const random = createRandom(0x0badf00d);
+ const mismatches: Array<string> = [];
+ for (let index = 0; index < 500; index++) {
+ const value = randomString(random, 30);
+ if (canonicalizeStoredPhrase(value) !== legacyCanonicalize(value)) {
+ mismatches.push(JSON.stringify(value));
+ }
+ }
+ expect(mismatches).toEqual([]);
+ });
+});
+
+describe('SubstringMatcher', () => {
+ test('is null only for an empty pattern set', () => {
+ expect(SubstringMatcher.fromPatterns([])).toBeNull();
+ expect(SubstringMatcher.fromPatterns(new Set<string>())).toBeNull();
+ expect(SubstringMatcher.fromPatterns([''])).not.toBeNull();
+ });
+
+ test('agrees with String.includes on generated patterns and texts', () => {
+ const random = createRandom(0x2468ace0);
+ const mismatches: Array<string> = [];
+ for (let round = 0; round < 500; round++) {
+ const patternCount = 1 + Math.floor(random() * 6);
+ const patterns: Array<string> = [];
+ for (let index = 0; index < patternCount; index++) {
+ patterns.push(randomString(random, 5));
+ }
+ const matcher = SubstringMatcher.fromPatterns(patterns);
+ if (matcher === null) {
+ mismatches.push(`unexpected null for ${JSON.stringify(patterns)}`);
+ continue;
+ }
+ for (let index = 0; index < 10; index++) {
+ const text = randomString(random, 30);
+ const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern));
+ const actual = matcher.test(text);
+ if (expected !== actual) {
+ mismatches.push(`${JSON.stringify(patterns)} / ${JSON.stringify(text)}: ${expected} vs ${actual}`);
+ }
+ }
+ }
+ expect(mismatches).toEqual([]);
+ });
+
+ test('an empty pattern matches any non-empty text', () => {
+ const matcher = SubstringMatcher.fromPatterns(['', 'zzz']);
+ expect(matcher).not.toBeNull();
+ expect(matcher!.test('')).toBe(false);
+ expect(matcher!.test('anything')).toBe(true);
+ });
+
+ test('finds patterns that are suffixes of other pattern prefixes', () => {
+ const matcher = SubstringMatcher.fromPatterns(['abcd', 'bc']);
+ expect(matcher!.test('xabcx')).toBe(true);
+ expect(matcher!.test('abd')).toBe(false);
+ expect(matcher!.test('zabcdz')).toBe(true);
+ });
+
+ test('scales to a very large pattern set without a compile limit', () => {
+ const random = createRandom(0x13579bdf);
+ const patterns: Array<string> = [];
+ for (let index = 0; index < 20000; index++) {
+ patterns.push(`${randomString(random, 4)}${index}`);
+ }
+ const matcher = SubstringMatcher.fromPatterns(patterns);
+ expect(matcher).not.toBeNull();
+ const mismatches: Array<string> = [];
+ for (let index = 0; index < 100; index++) {
+ const text = index % 2 === 0 ? randomString(random, 400) : `${randomString(random, 50)}${patterns[index]!}`;
+ const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern));
+ const actual = matcher!.test(text);
+ if (expected !== actual) {
+ mismatches.push(`${JSON.stringify(text)}: ${expected} vs ${actual}`);
+ }
+ }
+ expect(mismatches).toEqual([]);
+ });
+});
diff --git a/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts b/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts
index b6b66fccf..f267c953d 100644
--- a/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts
+++ b/fluxer_api/src/api/utils/PhraseBlocklistNormalization.ts
@@ -11,11 +11,11 @@ const WHITESPACE_REGEX = /\s+/gu;
const MIN_AGGRESSIVE_FORM_LENGTH = 3;
interface PhraseMatchForms {
- raw: string;
- words: string;
- compact: string;
- asciiWords: string;
- asciiCompact: string;
+ readonly raw: string;
+ readonly words: string;
+ readonly compact: string;
+ readonly asciiWords: string;
+ readonly asciiCompact: string;
}
function collapseWhitespace(value: string): string {
@@ -46,21 +46,50 @@ function buildAsciiWordFormFromCanonical(value: string): string {
);
}
+class LazyPhraseMatchForms implements PhraseMatchForms {
+ readonly raw: string;
+ private wordsForm: string | null = null;
+ private compactForm: string | null = null;
+ private asciiWordsForm: string | null = null;
+ private asciiCompactForm: string | null = null;
+
+ constructor(value: string) {
+ this.raw = canonicalizeStoredPhrase(value);
+ }
+
+ get words(): string {
+ if (this.wordsForm === null) {
+ this.wordsForm = maybeKeepAggressiveForm(buildWordFormFromCanonical(this.raw));
+ }
+ return this.wordsForm;
+ }
+
+ get compact(): string {
+ if (this.compactForm === null) {
+ this.compactForm = maybeKeepAggressiveForm(this.words.replace(WHITESPACE_REGEX, ''));
+ }
+ return this.compactForm;
+ }
+
+ get asciiWords(): string {
+ if (this.asciiWordsForm === null) {
+ this.asciiWordsForm = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(this.raw));
+ }
+ return this.asciiWordsForm;
+ }
+
+ get asciiCompact(): string {
+ if (this.asciiCompactForm === null) {
+ this.asciiCompactForm = maybeKeepAggressiveForm(this.asciiWords.replace(WHITESPACE_REGEX, ''));
+ }
+ return this.asciiCompactForm;
+ }
+}
+
export function canonicalizeStoredPhrase(value: string): string {
return stripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim();
}
export function buildPhraseMatchForms(value: string): PhraseMatchForms {
- const raw = canonicalizeStoredPhrase(value);
- const words = maybeKeepAggressiveForm(buildWordFormFromCanonical(raw));
- const compact = maybeKeepAggressiveForm(words.replace(WHITESPACE_REGEX, ''));
- const asciiWords = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(raw));
- const asciiCompact = maybeKeepAggressiveForm(asciiWords.replace(WHITESPACE_REGEX, ''));
- return {
- raw,
- words,
- compact,
- asciiWords,
- asciiCompact,
- };
+ return new LazyPhraseMatchForms(value);
}
diff --git a/fluxer_api/src/api/utils/RequestJsonBody.ts b/fluxer_api/src/api/utils/RequestJsonBody.ts
new file mode 100644
index 000000000..e9b03774a
--- /dev/null
+++ b/fluxer_api/src/api/utils/RequestJsonBody.ts
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {HonoRequest} from 'hono';
+import {parseJsonPreservingLargeIntegers} from './LosslessJsonParser';
+
+interface RequestJsonBody {
+ parsed: boolean;
+ value: unknown;
+}
+
+const bodyCache = new WeakMap<Request, RequestJsonBody>();
+
+export async function readRequestJsonBody(req: HonoRequest): Promise<RequestJsonBody> {
+ const raw = req.raw;
+ const cached = bodyCache.get(raw);
+ if (cached) return cached;
+ let body: RequestJsonBody;
+ try {
+ const text = await req.text();
+ body = {parsed: true, value: text.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(text)};
+ } catch {
+ body = {parsed: false, value: {}};
+ }
+ bodyCache.set(raw, body);
+ return body;
+}
diff --git a/fluxer_api/src/api/utils/SubstringMatcher.ts b/fluxer_api/src/api/utils/SubstringMatcher.ts
new file mode 100644
index 000000000..22083297a
--- /dev/null
+++ b/fluxer_api/src/api/utils/SubstringMatcher.ts
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+const ROOT_NODE = 0;
+const ASCII_ROOT_SIZE = 128;
+const NO_TRANSITION = -1;
+const EDGE_KEY_STRIDE = 65536;
+
+export class SubstringMatcher {
+ private readonly rowStart: Int32Array;
+ private readonly edgeCodes: Int32Array;
+ private readonly edgeTargets: Int32Array;
+ private readonly failures: Int32Array;
+ private readonly terminals: Uint8Array;
+ private readonly asciiRoot: Int32Array;
+ private readonly matchesEverything: boolean;
+
+ private constructor(
+ rowStart: Int32Array,
+ edgeCodes: Int32Array,
+ edgeTargets: Int32Array,
+ failures: Int32Array,
+ terminals: Uint8Array,
+ matchesEverything: boolean,
+ ) {
+ this.rowStart = rowStart;
+ this.edgeCodes = edgeCodes;
+ this.edgeTargets = edgeTargets;
+ this.failures = failures;
+ this.terminals = terminals;
+ this.matchesEverything = matchesEverything;
+ this.asciiRoot = new Int32Array(ASCII_ROOT_SIZE);
+ for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) {
+ const code = edgeCodes[edge]!;
+ if (code < ASCII_ROOT_SIZE) {
+ this.asciiRoot[code] = edgeTargets[edge]!;
+ }
+ }
+ }
+
+ static fromPatterns(patterns: Iterable<string>): SubstringMatcher | null {
+ const children = new Map<number, number>();
+ const edgeSources: Array<number> = [];
+ const edgeLabels: Array<number> = [];
+ const edgeChildren: Array<number> = [];
+ const terminalNodes: Array<number> = [];
+ let nodeCount = 1;
+ let matchesEverything = false;
+ let count = 0;
+ for (const pattern of patterns) {
+ count++;
+ if (pattern.length === 0) {
+ matchesEverything = true;
+ continue;
+ }
+ let node = ROOT_NODE;
+ for (let index = 0; index < pattern.length; index++) {
+ const code = pattern.charCodeAt(index);
+ const key = node * EDGE_KEY_STRIDE + code;
+ let next = children.get(key);
+ if (next === undefined) {
+ next = nodeCount++;
+ children.set(key, next);
+ edgeSources.push(node);
+ edgeLabels.push(code);
+ edgeChildren.push(next);
+ }
+ node = next;
+ }
+ terminalNodes.push(node);
+ }
+ if (count === 0) {
+ return null;
+ }
+ children.clear();
+ const edgeCount = edgeSources.length;
+ const rowStart = new Int32Array(nodeCount + 1);
+ for (let edge = 0; edge < edgeCount; edge++) {
+ rowStart[edgeSources[edge]! + 1]!++;
+ }
+ for (let node = 0; node < nodeCount; node++) {
+ rowStart[node + 1]! += rowStart[node]!;
+ }
+ const cursor = rowStart.slice(0, nodeCount);
+ const edgeCodes = new Int32Array(edgeCount);
+ const edgeTargets = new Int32Array(edgeCount);
+ for (let edge = 0; edge < edgeCount; edge++) {
+ const source = edgeSources[edge]!;
+ const slot = cursor[source]!;
+ cursor[source] = slot + 1;
+ edgeCodes[slot] = edgeLabels[edge]!;
+ edgeTargets[slot] = edgeChildren[edge]!;
+ }
+ for (let node = 0; node < nodeCount; node++) {
+ const start = rowStart[node]!;
+ const end = rowStart[node + 1]!;
+ for (let index = start + 1; index < end; index++) {
+ const code = edgeCodes[index]!;
+ const target = edgeTargets[index]!;
+ let scan = index - 1;
+ while (scan >= start && edgeCodes[scan]! > code) {
+ edgeCodes[scan + 1] = edgeCodes[scan]!;
+ edgeTargets[scan + 1] = edgeTargets[scan]!;
+ scan--;
+ }
+ edgeCodes[scan + 1] = code;
+ edgeTargets[scan + 1] = target;
+ }
+ }
+ const failures = new Int32Array(nodeCount);
+ const terminals = new Uint8Array(nodeCount);
+ for (const node of terminalNodes) {
+ terminals[node] = 1;
+ }
+ const queue = new Int32Array(nodeCount);
+ let tail = 0;
+ for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) {
+ queue[tail] = edgeTargets[edge]!;
+ tail++;
+ }
+ for (let head = 0; head < tail; head++) {
+ const node = queue[head]!;
+ if (terminals[failures[node]!]! === 1) {
+ terminals[node] = 1;
+ }
+ for (let edge = rowStart[node]!; edge < rowStart[node + 1]!; edge++) {
+ const code = edgeCodes[edge]!;
+ const child = edgeTargets[edge]!;
+ let candidate = failures[node]!;
+ let target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code);
+ while (target === NO_TRANSITION && candidate !== ROOT_NODE) {
+ candidate = failures[candidate]!;
+ target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code);
+ }
+ failures[child] = target === NO_TRANSITION ? ROOT_NODE : target;
+ queue[tail] = child;
+ tail++;
+ }
+ }
+ return new SubstringMatcher(rowStart, edgeCodes, edgeTargets, failures, terminals, matchesEverything);
+ }
+
+ test(text: string): boolean {
+ if (text.length === 0) {
+ return false;
+ }
+ if (this.matchesEverything) {
+ return true;
+ }
+ const rowStart = this.rowStart;
+ const edgeCodes = this.edgeCodes;
+ const edgeTargets = this.edgeTargets;
+ const failures = this.failures;
+ const terminals = this.terminals;
+ const asciiRoot = this.asciiRoot;
+ let node = ROOT_NODE;
+ for (let index = 0; index < text.length; index++) {
+ const code = text.charCodeAt(index);
+ if (node === ROOT_NODE && code < ASCII_ROOT_SIZE) {
+ node = asciiRoot[code]!;
+ } else {
+ let next = findTransition(rowStart, edgeCodes, edgeTargets, node, code);
+ while (next === NO_TRANSITION && node !== ROOT_NODE) {
+ node = failures[node]!;
+ next = findTransition(rowStart, edgeCodes, edgeTargets, node, code);
+ }
+ node = next === NO_TRANSITION ? ROOT_NODE : next;
+ }
+ if (terminals[node]! === 1) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
+
+function findTransition(
+ rowStart: Int32Array,
+ edgeCodes: Int32Array,
+ edgeTargets: Int32Array,
+ node: number,
+ code: number,
+): number {
+ let low = rowStart[node]!;
+ let high = rowStart[node + 1]! - 1;
+ while (low <= high) {
+ const middle = (low + high) >>> 1;
+ const candidate = edgeCodes[middle]!;
+ if (candidate === code) {
+ return edgeTargets[middle]!;
+ }
+ if (candidate < code) {
+ low = middle + 1;
+ } else {
+ high = middle - 1;
+ }
+ }
+ return NO_TRANSITION;
+}