Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

16.7 KB
Raw Blame
import { constants } from "node:fs";
import {
  lstat,
  mkdtemp,
  open,
  readFile,
  readdir,
  realpath,
  rm,
  stat,
  writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, extname, join, resolve } from "node:path";
import { unzipSync } from "fflate";
import { SaxesParser } from "saxes";
import { ConfigurationError, errorMessage } from "./errors.js";
import { checkDocumentPath, type DocumentRootSource } from "./prompt-files.js";
import { expandHome } from "./runtime.js";
import {
  documentSourceRoot,
  gitMarkerRoot,
  isGitMetadataDirectory,
  nullIfMissingFile,
} from "./targets.js";

const DOCUMENT_EXTENSIONS = new Set([
  ".md",
  ".markdown",
  ".txt",
  ".pdf",
  ".docx",
]);

export interface PreparedKnowledgeBase {
  path: string;
  sources: string[];
  protectedRoots: string[];
  cleanup(): Promise<void>;
}

export interface KnowledgeBaseSnapshot {
  readonly sources: readonly string[];
  readonly protectedRoots?: readonly string[];
  readonly documents: Readonly<Record<string, string>>;
}

export async function knowledgeBaseProtectedRoots(
  snapshot: KnowledgeBaseSnapshot,
  signal?: AbortSignal,
): Promise<string[]> {
  return snapshot.protectedRoots === undefined
    ? await Promise.all(
        snapshot.sources.map(
          async (source) =>
            (await gitMarkerRoot(source, signal, "outermost")) ?? source,
        ),
      )
    : [...snapshot.protectedRoots];
}

/** @internal Extract once so campaign identity and workers use identical inputs. */
export async function readKnowledgeBaseSnapshot(
  paths: readonly string[],
  signal?: AbortSignal,
  roots: string | DocumentRootSource = [],
): Promise<KnowledgeBaseSnapshot> {
  try {
    const sources = new Set<string>();
    const documents = new Set<string>();
    for (const requested of paths) {
      signal?.throwIfAborted();
      if (!requested.trim())
        throw new Error("Knowledge base paths cannot be empty.");
      const path = resolve(expandHome(requested));
      const metadata = await lstat(path);
      if (metadata.isSymbolicLink()) {
        throw new Error(
          `Knowledge base paths cannot be symbolic links: ${path}`,
        );
      }
      if (!metadata.isFile() && !metadata.isDirectory()) {
        throw new Error(
          `Knowledge base path is not a file or directory: ${path}`,
        );
      }

      const source = await realpath(path);
      if (path !== source) {
        if (typeof roots === "string")
          roots = [await documentSourceRoot(roots, signal)];
        await checkDocumentPath(path, source, roots, signal);
      }
      const selected = metadata.isDirectory()
        ? (await discover(source, signal)).sort()
        : [source];
      if (selected.length === 0) {
        throw new Error(
          `Knowledge base directory contains no supported documents: ${path}`,
        );
      }
      for (const document of selected) {
        documents.add(document);
      }
      sources.add(source);
    }

    const extracted: Record<string, string> = {};
    let index = 0;
    for (const document of documents) {
      signal?.throwIfAborted();
      const metadata = await lstat(document);
      if (process.platform !== "win32" && (metadata.mode & 0o444) === 0) {
        throw new Error(`Knowledge base document is not readable: ${document}`);
      }
      const bytes = await readFile(document, {
        flag: constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
        signal,
      });
      const extension = extname(document).toLowerCase();
      const text =
        extension === ".pdf"
          ? await extractPdf(document, bytes, signal)
          : extension === ".docx"
            ? extractDocx(document, bytes)
            : decodeText(document, bytes);
      if ((extension === ".pdf" || extension === ".docx") && !text.trim()) {
        throw new Error(
          `Knowledge base document contains no extractable text: ${document}`,
        );
      }
      const name = `${index}-${basename(document)}.txt`;
      // The prefix and suffix can exceed the filesystem's 255-byte name limit.
      const filename = Buffer.byteLength(name) > 255 ? `${index}.txt` : name;
      extracted[filename] = text;
      index++;
    }
    const snapshot = {
      sources: [...sources],
      documents: extracted,
    };
    return {
      ...snapshot,
      protectedRoots: await knowledgeBaseProtectedRoots(snapshot, signal),
    };
  } catch (error) {
    if (signal?.aborted || error instanceof ConfigurationError) throw error;
    throw new ConfigurationError(errorMessage(error), { cause: error });
  }
}

export async function prepareKnowledgeBase(
  input: readonly string[] | KnowledgeBaseSnapshot,
  signal?: AbortSignal,
  directory?: string,
  repository?: string,
): Promise<PreparedKnowledgeBase> {
  try {
    const snapshot =
      "documents" in input
        ? input
        : await readKnowledgeBaseSnapshot(input, signal, repository);
    const protectedRoots = await knowledgeBaseProtectedRoots(snapshot, signal);
    const path = await mkdtemp(
      join(await realpath(directory ?? tmpdir()), "codex-security-knowledge-"),
    );
    try {
      for (const [filename, text] of Object.entries(snapshot.documents)) {
        signal?.throwIfAborted();
        await writeFile(join(path, filename), text, {
          encoding: "utf8",
          mode: 0o600,
          signal,
        });
      }
    } catch (error) {
      await rm(path, { recursive: true, force: true });
      throw error;
    }
    return {
      path,
      sources: [...snapshot.sources],
      protectedRoots,
      cleanup: () => rm(path, { recursive: true, force: true }),
    };
  } catch (error) {
    if (signal?.aborted || error instanceof ConfigurationError) throw error;
    throw new ConfigurationError(errorMessage(error), { cause: error });
  }
}

async function discover(
  directory: string,
  signal?: AbortSignal,
): Promise<string[]> {
  signal?.throwIfAborted();
  const documents: string[] = [];
  const entries = await readdir(directory, { withFileTypes: true });
  signal?.throwIfAborted();
  for (const entry of entries) {
    signal?.throwIfAborted();
    if (
      (process.platform === "win32" ? entry.name.toLowerCase() : entry.name) ===
      ".git"
    )
      continue;
    const path = join(directory, entry.name);
    if (entry.name.toLowerCase() === ".git") {
      const marker = await stat(join(directory, ".git"), {
        bigint: true,
      }).catch(nullIfMissingFile);
      if (marker !== null) {
        const candidate = await lstat(path, { bigint: true });
        if (candidate.dev === marker.dev && candidate.ino === marker.ino) {
          continue;
        }
      }
      if (entry.isDirectory() && (await isGitMetadataDirectory(path, signal))) {
        continue;
      }
    }
    if (entry.isDirectory()) {
      for (const document of await discover(path, signal)) {
        documents.push(document);
      }
    } else if (entry.isFile()) {
      // Office keeps a "~$" owner file next to each open document.
      if (entry.name.startsWith("~$")) continue;
      if (entry.name.startsWith("._") && (await isAppleDouble(path))) continue;
      // LibreOffice keeps a ".~lock.<name>#" file next to each open document.
      if (/^\.~lock\..+#$/s.test(entry.name)) continue;
      if (!DOCUMENT_EXTENSIONS.has(extname(path).toLowerCase())) {
        const bytes = await readFile(path, {
          flag: constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
          signal,
        });
        try {
          decodeText(path, bytes);
        } catch {
          continue;
        }
      }
      documents.push(path);
    }
  }
  return documents;
}

// AppleDouble sidecars and ordinary documents can share the same "._" prefix.
async function isAppleDouble(path: string): Promise<boolean> {
  const file = await open(
    path,
    constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
  );
  try {
    const header = Buffer.alloc(4);
    const { bytesRead } = await file.read(header, 0, header.length, 0);
    return bytesRead === header.length && header.readUInt32BE() === 0x00051607;
  } finally {
    await file.close();
  }
}

function decodeText(path: string, bytes: Uint8Array): string {
  if (bytes.includes(0)) {
    throw new Error(`Knowledge base document contains binary data: ${path}`);
  }
  try {
    return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
  } catch (error) {
    throw new Error(`Knowledge base document is not valid UTF-8: ${path}`, {
      cause: error,
    });
  }
}

async function extractPdf(
  path: string,
  bytes: Uint8Array,
  signal?: AbortSignal,
): Promise<string> {
  try {
    const { getDocument, VerbosityLevel } =
      await import("pdfjs-dist/legacy/build/pdf.mjs");
    const loadingTask = getDocument({
      data: new Uint8Array(bytes),
      stopAtErrors: true,
      verbosity: VerbosityLevel.ERRORS,
    });
    try {
      const document = await loadingTask.promise;
      signal?.throwIfAborted();
      const pages: string[] = [];
      for (let number = 1; number <= document.numPages; number++) {
        signal?.throwIfAborted();
        const page = await document.getPage(number);
        signal?.throwIfAborted();
        const content = await page.getTextContent();
        signal?.throwIfAborted();
        const parts: string[] = [];
        for (const item of content.items) {
          if (!("str" in item)) continue;
          // Separate runs when the PDF supplies no whitespace, including across forms.
          if (/^\S/u.test(item.str) && /\S$/u.test(parts.at(-1) ?? "")) {
            parts.push(" ");
          }
          if (item.str) parts.push(item.str);
          if (item.hasEOL) parts.push("\n");
        }
        pages.push(parts.join(""));
      }
      return pages.join("\n");
    } finally {
      await loadingTask.destroy();
      signal?.throwIfAborted();
    }
  } catch (error) {
    signal?.throwIfAborted();
    throw new Error(`Cannot extract text from knowledge base PDF: ${path}`, {
      cause: error,
    });
  }
}

function extractDocx(path: string, bytes: Uint8Array): string {
  try {
    const files = unzipSync(bytes, {
      filter: (file) => {
        if (file.name !== "word/document.xml") return false;
        if (file.originalSize > 25 * 1024 * 1024) {
          throw new Error("DOCX document text exceeds 25 MB.");
        }
        return true;
      },
    });
    const document = files["word/document.xml"];
    if (document === undefined) throw new Error("Missing word/document.xml.");
    const xml = decodeText(path, document);
    let text: string[] = [];
    let fields: {
      instructions: boolean;
      literal: string[];
      result: string[];
    }[] = [];
    const stories: { text: string[]; fields: typeof fields }[] = [];
    const parser = new SaxesParser();
    const positions = new Set(["align", "posOffset"]);
    const drawingLayout = new Map([
      [
        "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
        positions,
      ],
      ["http://purl.oclc.org/ooxml/drawingml/wordprocessingDrawing", positions],
      [
        "http://schemas.microsoft.com/office/word/2010/wordprocessingDrawing",
        new Set(["pctPosHOffset", "pctPosVOffset", "pctWidth", "pctHeight"]),
      ],
    ]);
    const declarations = new Map<string, (Set<string> | undefined)[]>();
    const layout: boolean[] = [];
    let documentElement = false;
    let ignoredDepth = 0;
    let instructionText = false;
    // Paragraph properties list tab stops as w:tab elements inside w:tabs.
    let tabStops = false;
    // Content controls mark unfilled prompt text with w:showingPlcHdr.
    const placeholders: boolean[] = [];
    const wordAttribute = (
      attributes: Record<string, string>,
      localName: string,
    ) =>
      decodeXml(
        Object.entries(attributes).find(
          ([key]) => key.split(":").at(-1) === localName,
        )?.[1] ?? "",
      );
    const append = (value: string) => {
      if (ignoredDepth > 0) return;
      const field = fields.at(-1);
      if (field === undefined) text.push(value);
      else if (!field.instructions) field.result.push(value);
      else if (!instructionText) field.literal.push(value);
    };
    const finishField = (closed: boolean) => {
      const field = fields.pop();
      if (field !== undefined) {
        // Unclosed fields retain ordinary runs; completed fields show only results.
        append((closed ? "" : field.literal.join("")) + field.result.join(""));
      }
    };
    parser.on("attribute", ({ name, value }) => {
      if (name !== "xmlns" && !name.startsWith("xmlns:")) return;
      const bindings = declarations.get(name) ?? [];
      bindings.push(drawingLayout.get(decodeXml(value)));
      declarations.set(name, bindings);
    });
    parser.on("opentag", ({ name, attributes }) => {
      const colon = name.indexOf(":");
      const declaration = colon < 0 ? "xmlns" : "xmlns:" + name.slice(0, colon);
      const localName = name.split(":").at(-1)!;
      layout.push(
        declarations.get(declaration)?.at(-1)?.has(localName) ?? false,
      );
      if (
        ignoredDepth > 0 ||
        localName === "del" ||
        localName === "delText" ||
        localName === "delInstrText" ||
        localName === "moveFrom" ||
        (localName === "sdtContent" && placeholders.at(-1))
      ) {
        ignoredDepth++;
        return;
      }
      if (localName === "sdt") placeholders.push(false);
      else if (localName === "showingPlcHdr" && placeholders.length)
        placeholders[placeholders.length - 1] = !["0", "false", "off"].includes(
          wordAttribute(attributes, "val"),
        );
      if (localName === "txbxContent") {
        // Text boxes have their own field boundaries.
        stories.push({ text, fields });
        text = [];
        fields = [];
      } else if (localName === "document") documentElement = true;
      else if (localName === "instrText") instructionText = true;
      else if (localName === "fldChar") {
        const type = wordAttribute(attributes, "fldCharType");
        if (type === "begin")
          fields.push({ instructions: true, literal: [], result: [] });
        else if (type === "separate" && fields.length > 0)
          fields.at(-1)!.instructions = false;
        else if (type === "end") finishField(true);
      } else if (localName === "br" || localName === "cr") append("\n");
      else if (localName === "tabs") tabStops = true;
      else if (localName === "tab" && !tabStops) append("\t");
      else if (localName === "noBreakHyphen") append("-");
    });
    parser.on("closetag", ({ name, attributes, isSelfClosing }) => {
      layout.pop();
      for (const key of Object.keys(attributes)) {
        const bindings = declarations.get(key);
        if (!bindings) continue;
        bindings.pop();
        if (!bindings.length) declarations.delete(key);
      }
      const localName = name.split(":").at(-1);
      if (ignoredDepth > 0) {
        ignoredDepth--;
        return;
      }
      if (localName === "sdt") placeholders.pop();
      if (localName === "txbxContent") {
        while (fields.length > 0) finishField(false);
        const content = text.join("");
        ({ text, fields } = stories.pop()!);
        append(content);
      }
      if (localName === "instrText") instructionText = false;
      if (localName === "tabs") tabStops = false;
      if (!isSelfClosing && localName === "p") append("\n");
    });
    // Word stores drawing positions and relative sizes as character data.
    const appendCharacters = (value: string) => {
      if (!layout.at(-1)) append(value);
    };
    parser.on("text", (value) => appendCharacters(decodeXml(value)));
    parser.on("cdata", (value) =>
      appendCharacters(value.replaceAll("&amp;", "&")),
    );
    // Keep the existing entity decoder's handling of unusable references.
    // CDATA is literal text, so only undo this escaping in its handler.
    parser.write(xml.replaceAll("&", "&amp;")).close();
    if (!documentElement) throw new Error("Malformed word/document.xml.");
    while (fields.length > 0) finishField(false);
    return text.join("");
  } catch (error) {
    throw new Error(`Cannot extract text from knowledge base DOCX: ${path}`, {
      cause: error,
    });
  }
}

function decodeXml(value: string): string {
  const entities: Record<string, string> = {
    amp: "&",
    lt: "<",
    gt: ">",
    quot: '"',
    apos: "'",
  };
  return value.replace(
    /&(amp|lt|gt|quot|apos|#\d+|#x[\da-f]+);/giu,
    (entity, name: string) => {
      if (!name.startsWith("#")) return entities[name.toLowerCase()] ?? entity;
      const hexadecimal = name[1]?.toLowerCase() === "x";
      const codePoint = Number.parseInt(
        name.slice(hexadecimal ? 2 : 1),
        hexadecimal ? 16 : 10,
      );
      if (
        codePoint > 0x10ffff ||
        (codePoint >= 0xd800 && codePoint <= 0xdfff)
      ) {
        return entity;
      }
      return String.fromCodePoint(codePoint);
    },
  );
}