Skip to content

Linger is An Open-Source Personal Agent That Runs Fully on Your PC. We aim to break Big Tech’s grip on AI by bringing powerful local AI to everyday computers through efficient model–harness co-design, so everyone can run their own.

8.9 KB
Raw Blame
import { MIMEType } from "node:util";

import {
  DocumentNormalizationError,
  type DocumentNormalizationInput,
} from "./document-normalizer.js";

export type DocumentSourceKind =
  | "markdown"
  | "plain"
  | "html"
  | "csv"
  | "tsv"
  | "json"
  | "xml"
  | "yaml"
  | "code"
  | "pdf"
  | "doc"
  | "docx"
  | "xlsx"
  | "pptx"
  | "odt"
  | "ods"
  | "odp"
  | "epub"
  | "mail"
  | "zip"
  | "gzip"
  | "svg"
  | "unsupported";

const TEXT_DOCUMENT_SOURCE_KINDS = new Set<DocumentSourceKind>([
  "markdown",
  "plain",
  "html",
  "csv",
  "tsv",
  "json",
  "xml",
  "yaml",
  "code",
  "svg",
]);

const ZIP_DOCUMENT_KINDS = new Set<DocumentSourceKind>([
  "docx",
  "xlsx",
  "pptx",
  "odt",
  "ods",
  "odp",
  "epub",
]);

const MEDIA_TYPE_KINDS: Record<string, DocumentSourceKind> = {
  "application/epub+zip": "epub",
  "application/vnd.oasis.opendocument.presentation": "odp",
  "application/vnd.oasis.opendocument.spreadsheet": "ods",
  "application/vnd.oasis.opendocument.text": "odt",
  "application/vnd.openxmlformats-officedocument.wordprocessingml.template":
    "docx",
  "application/vnd.openxmlformats-officedocument.presentationml.template":
    "pptx",
  "application/vnd.openxmlformats-officedocument.presentationml.slideshow":
    "pptx",
  "application/vnd.openxmlformats-officedocument.spreadsheetml.template":
    "xlsx",
  "message/rfc822": "mail",
  "multipart/related": "mail",
  "application/x-mimearchive": "mail",
  "application/csv": "csv",
  "application/ecmascript": "code",
  "application/javascript": "code",
  "application/json": "json",
  "application/ld+json": "json",
  "application/msword": "doc",
  "application/pdf": "pdf",
  "application/sql": "code",
  "application/toml": "code",
  "application/vnd.ms-word": "doc",
  "application/vnd.openxmlformats-officedocument.presentationml.presentation":
    "pptx",
  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
  "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
    "docx",
  "application/gzip": "gzip",
  "application/x-gzip": "gzip",
  "application/x-javascript": "code",
  "application/x-msword": "doc",
  "application/x-yaml": "yaml",
  "application/x-zip": "zip",
  "application/x-zip-compressed": "zip",
  "application/xhtml+xml": "html",
  "application/xml": "xml",
  "application/yaml": "yaml",
  "application/zip": "zip",
  "image/svg+xml": "svg",
  "text/comma-separated-values": "csv",
  "text/css": "code",
  "text/csv": "csv",
  "text/html": "html",
  "text/javascript": "code",
  "text/json": "json",
  "text/markdown": "markdown",
  "text/plain": "plain",
  "text/tab-separated-values": "tsv",
  "text/tsv": "tsv",
  "text/x-markdown": "markdown",
  "text/x-python": "code",
  "text/x-sh": "code",
  "text/x-sql": "code",
  "text/x-yaml": "yaml",
  "text/xml": "xml",
  "text/yaml": "yaml",
};

const EXPLICITLY_UNSUPPORTED_MEDIA_TYPES = new Set([
  "application/vnd.ms-excel",
  "application/vnd.ms-powerpoint",
  "application/vnd.rar",
  "application/x-7z-compressed",
  "application/x-rar-compressed",
  "application/x-tar",
]);

const EXTENSION_KINDS: Record<string, DocumentSourceKind> = {
  odt: "odt",
  ods: "ods",
  odp: "odp",
  epub: "epub",
  eml: "mail",
  mht: "mail",
  mhtml: "mail",
  dotx: "docx",
  potx: "pptx",
  ppsx: "pptx",
  xltx: "xlsx",
  c: "code",
  cpp: "code",
  css: "code",
  avif: "unsupported",
  bmp: "unsupported",
  csv: "csv",
  doc: "doc",
  docx: "docx",
  gif: "unsupported",
  go: "code",
  gz: "gzip",
  gzip: "gzip",
  h: "code",
  heic: "unsupported",
  heif: "unsupported",
  hpp: "code",
  htm: "html",
  html: "html",
  ico: "unsupported",
  java: "code",
  jpeg: "unsupported",
  jpg: "unsupported",
  js: "code",
  json: "json",
  jsx: "code",
  less: "code",
  log: "plain",
  markdown: "markdown",
  md: "markdown",
  mmd: "code",
  pdf: "pdf",
  png: "unsupported",
  pptx: "pptx",
  py: "code",
  rs: "code",
  scss: "code",
  sh: "code",
  sql: "code",
  svg: "svg",
  tif: "unsupported",
  tiff: "unsupported",
  toml: "code",
  ts: "code",
  tsv: "tsv",
  tsx: "code",
  txt: "plain",
  webp: "unsupported",
  xlsx: "xlsx",
  xml: "xml",
  yaml: "yaml",
  yml: "yaml",
  zip: "zip",
};

const REPRESENTATION_KINDS: Record<string, DocumentSourceKind> = {
  csv: "csv",
  doc: "doc",
  docx: "docx",
  gzip: "gzip",
  html: "html",
  json: "json",
  markdown: "markdown",
  pdf: "pdf",
  plain_text: "plain",
  png: "unsupported",
  pptx: "pptx",
  svg: "svg",
  tsv: "tsv",
  xlsx: "xlsx",
  xml: "xml",
  yaml: "yaml",
  zip: "zip",
};

const CODE_LANGUAGES: Record<string, string> = {
  c: "c",
  cpp: "cpp",
  css: "css",
  go: "go",
  h: "c",
  hpp: "cpp",
  java: "java",
  js: "javascript",
  jsx: "jsx",
  less: "less",
  mmd: "mermaid",
  py: "python",
  rs: "rust",
  scss: "scss",
  sh: "bash",
  sql: "sql",
  toml: "toml",
  ts: "typescript",
  tsx: "tsx",
};

export function resolveDocumentSourceKind(
  input: Pick<
    DocumentNormalizationInput,
    "filename" | "mediaType" | "representationFormat"
  >,
): DocumentSourceKind {
  const parsed = parseMediaType(input.mediaType);
  const fromMedia = kindFromMediaType(parsed.essence);
  if (fromMedia) {
    if (fromMedia === "zip") {
      const officeKind = zipDocumentKind(input);
      if (officeKind) return officeKind;
    }
    assertCompatibleCharset(fromMedia, parsed.charset);
    return fromMedia;
  }
  const fromName = kindFromFilename(input.filename);
  if (fromName) {
    assertCompatibleCharset(fromName, parsed.charset);
    return fromName;
  }
  const fromFormat = kindFromRepresentation(input.representationFormat);
  if (fromFormat) {
    assertCompatibleCharset(fromFormat, parsed.charset);
    return fromFormat;
  }
  if (parsed.essence.startsWith("text/")) {
    assertCompatibleCharset("plain", parsed.charset);
    return "plain";
  }
  return "unsupported";
}

export function isInProcessDocumentSource(
  input: Pick<
    DocumentNormalizationInput,
    "filename" | "mediaType" | "representationFormat"
  >,
): boolean {
  const kind = resolveDocumentSourceKind(input);
  // Structured parsing can block shared roles even when the source is text.
  return (
    kind !== "json" && kind !== "html" && TEXT_DOCUMENT_SOURCE_KINDS.has(kind)
  );
}

export function codeFenceLanguage(filename: string | undefined): string {
  const extension = fileExtension(filename);
  return CODE_LANGUAGES[extension] ?? extension ?? "text";
}

function kindFromMediaType(essence: string): DocumentSourceKind | undefined {
  if (!essence || essence === "application/octet-stream") return undefined;
  const mapped = MEDIA_TYPE_KINDS[essence];
  if (mapped) return mapped;
  if (
    essence.startsWith("image/") ||
    essence.startsWith("audio/") ||
    essence.startsWith("video/") ||
    EXPLICITLY_UNSUPPORTED_MEDIA_TYPES.has(essence)
  ) {
    return "unsupported";
  }
  return undefined;
}

function zipDocumentKind(
  input: Pick<DocumentNormalizationInput, "filename" | "representationFormat">,
): DocumentSourceKind | undefined {
  const fromName = kindFromFilename(input.filename);
  if (fromName && ZIP_DOCUMENT_KINDS.has(fromName)) return fromName;
  const fromFormat = kindFromRepresentation(input.representationFormat);
  if (fromFormat && ZIP_DOCUMENT_KINDS.has(fromFormat)) return fromFormat;
  return undefined;
}

function kindFromFilename(
  filename: string | undefined,
): DocumentSourceKind | undefined {
  const extension = fileExtension(filename);
  return extension ? EXTENSION_KINDS[extension] : undefined;
}

function kindFromRepresentation(
  representationFormat: string | undefined,
): DocumentSourceKind | undefined {
  const normalized = representationFormat?.trim().toLocaleLowerCase("en-US");
  if (!normalized) return undefined;
  return (
    REPRESENTATION_KINDS[normalized] ?? EXTENSION_KINDS[normalized] ?? undefined
  );
}

function parseMediaType(value: string): {
  essence: string;
  charset: string | null;
} {
  const separator = value.indexOf(";");
  const essence = (separator < 0 ? value : value.slice(0, separator))
    .trim()
    .toLocaleLowerCase("en-US");
  // Parse parameters independently so an absent or generic essence can still
  // fall back to the filename or representation format.
  const parameters = separator < 0 ? "" : value.slice(separator);
  const charset = new MIMEType(
    `application/octet-stream${parameters}`,
  ).params.get("charset");
  return { essence, charset };
}

function assertCompatibleCharset(
  kind: DocumentSourceKind,
  charset: string | null,
): void {
  if (!TEXT_DOCUMENT_SOURCE_KINDS.has(kind)) return;
  if (
    charset !== null &&
    !["utf-8", "utf8"].includes(charset.toLocaleLowerCase("en-US"))
  ) {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_UNSUPPORTED");
  }
}

function fileExtension(filename: string | undefined): string {
  if (!filename) return "";
  const basename = filename.split(/[/\\]/u).at(-1) ?? "";
  const separator = basename.lastIndexOf(".");
  if (separator <= 0 || separator === basename.length - 1) return "";
  return basename.slice(separator + 1).toLocaleLowerCase("en-US");
}