Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

964 B
Raw Blame
const asciiTokenPattern = /[a-z0-9_'-]+/gi;
const hanTokenPattern = /[\u3400-\u9fff]+/g;

function unique(tokens: string[]) {
  return Array.from(new Set(tokens.filter(Boolean)));
}

function tokenizeCjkText(input: string) {
  const source = input.toLowerCase();
  const tokens: string[] = [];

  for (const match of source.matchAll(asciiTokenPattern)) {
    tokens.push(match[0]);
  }

  for (const match of source.matchAll(hanTokenPattern)) {
    const text = match[0];
    tokens.push(text);

    for (const char of text) {
      tokens.push(char);
    }

    for (let index = 0; index < text.length - 1; index += 1) {
      tokens.push(text.slice(index, index + 2));
    }
  }

  return unique(tokens);
}

export function createCjkSearchTokenizer() {
  return {
    language: "cjk",
    normalizationCache: new Map<string, string>(),
    tokenize(raw: string) {
      if (typeof raw !== "string") return [raw];
      return tokenizeCjkText(raw);
    },
  };
}