Skip to content

Linger is An Open-Source Personal Agent That Runs Fully on Your PC. We aim to break Big Tech’s grip on AI by bringing powerful local AI to everyday computers through efficient model–harness co-design, so everyone can run their own.

8.7 KB
Raw Blame
import {
  DocumentNormalizationError,
  MAX_DOCUMENT_MARKDOWN_BYTES,
  MAX_DOCUMENT_SOURCE_BYTES,
  type DocumentNormalizationInput,
} from "./document-normalizer.js";
import {
  archiveEntryHeading,
  inflateGzip,
  listSafeZipFiles,
  stripGzipExtension,
} from "./document-archive-extract.js";
import {
  escapeMarkdownTableCell,
  extractDocText,
  extractOfficeMarkdown,
  extractSpreadsheetMarkdown,
  hasOleSignature,
  hasZipSignature,
} from "./document-office-extract.js";
import {
  codeFenceLanguage,
  isInProcessDocumentSource,
  resolveDocumentSourceKind,
} from "./document-source-kind.js";

export { isInProcessDocumentSource, resolveDocumentSourceKind };

const MAX_ARCHIVE_NESTING = 1;

function nextArchiveDepth(current: number | undefined): number {
  const depth = current ?? 0;
  if (depth >= MAX_ARCHIVE_NESTING) {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_UNSUPPORTED");
  }
  return depth + 1;
}

export async function normalizeDocumentContent(
  input: DocumentNormalizationInput,
  maxOutputBytes = MAX_DOCUMENT_MARKDOWN_BYTES,
): Promise<Uint8Array> {
  if (input.source.byteLength > MAX_DOCUMENT_SOURCE_BYTES) {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_TOO_LARGE");
  }
  const kind = resolveDocumentSourceKind(input);
  let markdown: string;
  switch (kind) {
    case "pdf":
      markdown = await (
        await import("./document-pdf-extract.js")
      ).extractPdfMarkdown(input.source);
      break;
    case "doc":
      markdown = hasZipSignature(input.source)
        ? await extractOfficeMarkdown(input.source, "docx")
        : await extractDocText(input.source);
      break;
    case "docx":
      markdown = hasOleSignature(input.source)
        ? await extractDocText(input.source)
        : await extractOfficeMarkdown(input.source, "docx");
      break;
    case "xlsx":
      markdown = await extractSpreadsheetMarkdown(input.source);
      break;
    case "pptx":
    case "odt":
    case "ods":
    case "odp":
    case "epub":
      markdown = await extractOfficeMarkdown(input.source, kind);
      break;
    case "mail":
      markdown = await (
        await import("./document-mail-extract.js")
      ).extractMailMarkdown(input.source);
      break;
    case "zip":
      markdown = await extractZipMarkdown(input);
      break;
    case "gzip":
      markdown = await extractGzipMarkdown(input);
      break;
    case "svg":
      markdown = (await import("./document-svg-extract.js")).svgToMarkdown(
        decodeUtf8Text(input.source),
        input.filename,
      );
      break;
    case "html":
      decodeUtf8Text(input.source);
      markdown = await extractOfficeMarkdown(input.source, "html");
      break;
    case "csv":
      markdown = delimitedToMarkdown(decodeUtf8Text(input.source), ",");
      break;
    case "tsv":
      markdown = delimitedToMarkdown(decodeUtf8Text(input.source), "\t");
      break;
    case "json":
      markdown = jsonToMarkdown(decodeUtf8Text(input.source));
      break;
    case "xml":
      markdown = fencedBlock("xml", decodeUtf8Text(input.source));
      break;
    case "yaml":
      markdown = fencedBlock("yaml", decodeUtf8Text(input.source));
      break;
    case "code":
      markdown = fencedBlock(
        codeFenceLanguage(input.filename),
        decodeUtf8Text(input.source),
      );
      break;
    case "markdown":
    case "plain":
      markdown = decodeUtf8Text(input.source);
      break;
    default:
      throw new DocumentNormalizationError("DOCUMENT_SOURCE_UNSUPPORTED");
  }
  return canonicalizeMarkdown(markdown, maxOutputBytes);
}

export function canonicalizeMarkdown(
  markdown: string,
  maxOutputBytes = MAX_DOCUMENT_MARKDOWN_BYTES,
): Uint8Array {
  const normalized = markdown
    .replace(/^\uFEFF/u, "")
    .replace(/\r\n?/gu, "\n")
    .normalize("NFC")
    .replace(/\n+$/u, "");
  if (!normalized.trim()) {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_EMPTY");
  }
  const encoded = new TextEncoder().encode(`${normalized}\n`);
  if (encoded.byteLength > maxOutputBytes) {
    throw new DocumentNormalizationError("DOCUMENT_MARKDOWN_TOO_LARGE");
  }
  return encoded;
}

async function extractZipMarkdown(
  input: DocumentNormalizationInput,
): Promise<string> {
  const nestedDepth = nextArchiveDepth(input.archiveDepth);
  const entries = await listSafeZipFiles(input.source);
  const sections: string[] = [];
  for (const entry of entries) {
    try {
      const markdown = await normalizeDocumentContent({
        archiveDepth: nestedDepth,
        filename: entry.filename,
        mediaType: "application/octet-stream",
        source: entry.bytes,
      });
      const body = new TextDecoder().decode(markdown).trim();
      if (!body) continue;
      sections.push(`## ${archiveEntryHeading(entry.filename)}\n\n${body}`);
    } catch (error) {
      if (!(error instanceof DocumentNormalizationError)) throw error;
      if (
        error.code === "DOCUMENT_SOURCE_TOO_LARGE" ||
        error.code === "DOCUMENT_MARKDOWN_TOO_LARGE"
      ) {
        throw error;
      }
    }
  }
  return sections.join("\n\n");
}

async function extractGzipMarkdown(
  input: DocumentNormalizationInput,
): Promise<string> {
  const nestedDepth = nextArchiveDepth(input.archiveDepth);
  const inflated = inflateGzip(input.source);
  return new TextDecoder().decode(
    await normalizeDocumentContent({
      archiveDepth: nestedDepth,
      filename: stripGzipExtension(input.filename),
      mediaType: "application/octet-stream",
      source: inflated,
    }),
  );
}

function decodeUtf8Text(source: Uint8Array): string {
  let decoded: string;
  try {
    decoded = new TextDecoder("utf-8", { fatal: true }).decode(source);
  } catch {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_INVALID");
  }
  if (decoded.includes("\u0000")) {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_INVALID");
  }
  return decoded;
}

function delimitedToMarkdown(text: string, delimiter: string): string {
  const rows = parseDelimitedRows(text, delimiter);
  if (rows.length === 0) {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_EMPTY");
  }
  const width = rows.reduce((max, row) => Math.max(max, row.length), 0);
  const padded = rows.map((row) =>
    Array.from({ length: width }, (_, index) =>
      escapeMarkdownTableCell(row[index] ?? ""),
    ),
  );
  const header = padded[0] ?? [];
  const body = padded.slice(1);
  return [
    `| ${header.join(" | ")} |`,
    `| ${header.map(() => "---").join(" | ")} |`,
    ...body.map((row) => `| ${row.join(" | ")} |`),
  ].join("\n");
}

function parseDelimitedRows(text: string, delimiter: string): string[][] {
  const rows: string[][] = [];
  let row: string[] = [];
  let field = "";
  let inQuotes = false;
  for (let index = 0; index < text.length; index += 1) {
    const character = text[index] ?? "";
    if (inQuotes) {
      if (character === '"') {
        if (text[index + 1] === '"') {
          field += '"';
          index += 1;
        } else {
          inQuotes = false;
        }
      } else {
        field += character;
      }
      continue;
    }
    if (character === '"') {
      inQuotes = true;
      continue;
    }
    if (character === delimiter) {
      row.push(field);
      field = "";
      continue;
    }
    if (character === "\n") {
      row.push(field);
      if (row.some((value) => value.trim())) rows.push(row);
      row = [];
      field = "";
      continue;
    }
    if (character === "\r") continue;
    field += character;
  }
  if (inQuotes) {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_INVALID");
  }
  if (field.length > 0 || row.length > 0) {
    row.push(field);
    if (row.some((value) => value.trim())) rows.push(row);
  }
  return rows;
}

function jsonToMarkdown(text: string): string {
  try {
    // Node 26 provides raw JSON values; the project's ES2022 types omit them.
    const json = JSON as typeof JSON & {
      rawJSON(source: string): unknown;
    };
    const parsed: unknown = JSON.parse(
      text,
      (_key: string, value: unknown, context?: { source?: string }): unknown =>
        typeof value === "number" && context?.source !== undefined
          ? json.rawJSON(context.source)
          : value,
    );
    return fencedBlock("json", JSON.stringify(parsed, null, 2));
  } catch {
    throw new DocumentNormalizationError("DOCUMENT_SOURCE_INVALID");
  }
}

function fencedBlock(language: string, body: string): string {
  const normalized = body.replace(/^\uFEFF/u, "").replace(/\n+$/u, "");
  let fenceLength = 3;
  for (const run of normalized.matchAll(/`+/gu)) {
    fenceLength = Math.max(fenceLength, run[0].length + 1);
  }
  const fence = "`".repeat(fenceLength);
  const info = /^[^`\r\n]*$/u.test(language) ? language : "text";
  return `${fence}${info}\n${normalized}\n${fence}`;
}