Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

2.9 KB
Raw Blame
import { Meta } from "..";
import { Document } from "../../../controllers/v2/types";
import { config } from "../../../config";
import { hasFormatOfType } from "../../../lib/format-utils";
import { isUsOnly } from "../../../lib/us-only";

export async function fetchProduct(
  meta: Meta,
  document: Document,
): Promise<Document> {
  if (!hasFormatOfType(meta.options.formats, "product")) {
    return document;
  }

  if (meta.internalOptions.zeroDataRetention) {
    document.warning =
      "Product format is not supported with zero data retention." +
      (document.warning ? " " + document.warning : "");
    return document;
  }

  if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {
    document.warning =
      "Product format is not supported for teams restricted to US-only processing." +
      (document.warning ? " " + document.warning : "");
    return document;
  }

  if (!config.PRODUCT_EXTRACTION_SERVICE_URL) {
    meta.logger.warn("PRODUCT_EXTRACTION_SERVICE_URL is not configured");
    document.warning =
      "Product format is not available (service not configured)." +
      (document.warning ? " " + document.warning : "");
    return document;
  }

  if (document.rawHtml === undefined) {
    throw new Error(
      "rawHtml is undefined -- fetchProduct must run after rawHtml is available",
    );
  }

  const url =
    document.metadata.url ??
    document.metadata.sourceURL ??
    meta.rewrittenUrl ??
    meta.url;

  const response = await fetch(
    `${config.PRODUCT_EXTRACTION_SERVICE_URL}/v1/extract-product`,
    {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ html: document.rawHtml, url }),
    },
  );

  if (!response.ok) {
    const error = await response
      .json()
      .catch(() => ({ detail: "Unknown error" }));
    throw new Error(`Product extraction failed: ${error.detail}`);
  }

  let data: any;
  try {
    data = await response.json();
  } catch {
    throw new Error(
      "Product extraction failed: service returned a non-JSON response",
    );
  }

  // The service contract is a 200 with a body of `{ product: Product | null }`.
  // A body that isn't an object, or is missing the `product` key entirely, is a
  // malformed response -- surface it as a service failure rather than silently
  // reporting "no product found" (which is reserved for an explicit `null`).
  if (typeof data !== "object" || data === null || !("product" in data)) {
    throw new Error(
      "Product extraction failed: service returned an unexpected response shape (missing 'product')",
    );
  }

  if (data.product) {
    document.product = data.product;
  } else {
    // `product` is null: the page loaded but is not a product page.
    document.warning =
      "No product found on this page; it does not appear to be a product page." +
      (document.warning ? " " + document.warning : "");
  }
  return document;
}