Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

7.8 KB
Raw Blame
import { config } from "../config";
import type { ScrapeOptions } from "../controllers/v2/types";
import { includesFormat } from "./format-utils";
import { getBigtableTable } from "./bigtable-client";
import { saltedUuidV7RowKey } from "./bigtable-row-key";
import { setSpanAttributes, withSpan } from "./otel-tracer";

const FAMILY = "f";
const FEEDBACK_ROW_RETENTION_MS = 24 * 60 * 60 * 1000;
/** Headroom past a configured feedback window longer than the default retention. */
const FEEDBACK_ROW_RETENTION_MARGIN_MS = 60 * 60 * 1000;
const QUALIFIER = "v";

type FeedbackEndpoint = "search" | "scrape" | "parse" | "map";
export type RefundClass =
  | "search"
  | "map"
  | "parse"
  | "scrape_basic"
  | "scrape_pdf"
  | "scrape_json"
  | "scrape_addon";

const REFUND_CLASSES: readonly RefundClass[] = [
  "search",
  "map",
  "parse",
  "scrape_basic",
  "scrape_pdf",
  "scrape_json",
  "scrape_addon",
];

type FeedbackJob = {
  requestId: string;
  teamId: string;
  refundClass: RefundClass;
  feedbackDeadlineMs: number;
  succeeded: boolean;
  creditsBilled: number;
  zeroDataRetention: boolean;
  /**
   * Search only. How many results each group returned, e.g.
   * `{"web":3,"images":0,"news":10}`, and which vertical served each position,
   * e.g. `{"web":{"1":"developer"}}`.
   *
   * Position feedback addresses a result as `{source, position}`, 1-indexed
   * within its group, so bounding a position needs the length of that group
   * and nothing else. They live here rather than beside the search row
   * because this is the record the feedback window is enforced from, and it
   * is the only one feedback reads.
   *
   * Absent on records written before these fields existed, on non-search
   * jobs, and under zero data retention. Absent is not empty: `{}` means the
   * search ran and every group was empty, which rejects every position.
   */
  numResultsBySource?: Record<string, number>;
  resultCategories?: Record<string, Record<string, string>>;
};

function parseFeedbackJob(value: Buffer | string): FeedbackJob {
  const parsed: unknown = JSON.parse(value.toString());
  const row = parsed as Record<string, unknown>;
  if (
    !parsed ||
    typeof parsed !== "object" ||
    row.version !== 1 ||
    typeof row.requestId !== "string" ||
    typeof row.teamId !== "string" ||
    typeof row.refundClass !== "string" ||
    !REFUND_CLASSES.includes(row.refundClass as RefundClass) ||
    typeof row.feedbackDeadlineMs !== "number" ||
    !Number.isFinite(row.feedbackDeadlineMs) ||
    typeof row.succeeded !== "boolean" ||
    typeof row.creditsBilled !== "number" ||
    !Number.isFinite(row.creditsBilled) ||
    typeof row.zeroDataRetention !== "boolean"
  ) {
    throw new Error("Invalid Bigtable feedback job row");
  }
  // The stored shape carries a `version` for forward compatibility; the
  // in-memory job does not, matching the scrape and extract state readers.
  const { version: _, ...job } = row;
  return job as FeedbackJob;
}

export async function readFeedbackJob(
  jobId: string,
): Promise<FeedbackJob | null> {
  const tableId = config.BIGTABLE_FEEDBACK_JOBS_TABLE;
  if (!tableId) return null;

  return withSpan("bigtable.feedback_job.read", async span => {
    setSpanAttributes(span, {
      "db.system": "bigtable",
      "bigtable.table": tableId,
      "bigtable.operation": "getRows",
    });
    const table = await getBigtableTable(tableId);
    const [rows] = await table.getRows({
      keys: [saltedUuidV7RowKey(jobId)],
      filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],
    });
    const cells = rows[0]?.data?.[FAMILY]?.[QUALIFIER];
    const cell = Array.isArray(cells) ? cells[0] : undefined;
    if (cell?.value == null) {
      setSpanAttributes(span, { "bigtable.read.outcome": "not_found" });
      return null;
    }

    const job = parseFeedbackJob(cell.value);
    if (job.feedbackDeadlineMs <= Date.now()) {
      setSpanAttributes(span, { "bigtable.read.outcome": "expired" });
      return job;
    }
    setSpanAttributes(span, {
      "bigtable.read.outcome": "found",
      "feedback.refund_class": job.refundClass,
    });
    return job;
  });
}

function scrapeRefundClass(options: ScrapeOptions): RefundClass {
  if (
    options.parsers?.some(
      parser =>
        parser === "pdf" ||
        (typeof parser === "object" && parser.type === "pdf"),
    )
  ) {
    return "scrape_pdf";
  }
  if (includesFormat(options.formats, "json")) return "scrape_json";
  if (
    includesFormat(options.formats, "screenshot") ||
    (options.actions?.length ?? 0) > 0
  ) {
    return "scrape_addon";
  }
  return "scrape_basic";
}

type FeedbackJobBase = {
  jobId: string;
  requestId: string;
  teamId: string;
  succeeded: boolean;
  creditsBilled: number;
  zeroDataRetention: boolean;
  completedAt?: Date;
  numResultsBySource?: Record<string, number>;
  resultCategories?: Record<string, Record<string, string>>;
};

type FeedbackJobWrite = FeedbackJobBase &
  (
    | { endpoint: "scrape"; scrapeOptions: ScrapeOptions }
    | { endpoint: Exclude<FeedbackEndpoint, "scrape"> }
  );

export async function writeFeedbackJob(
  params: FeedbackJobWrite,
): Promise<boolean> {
  const tableId = config.BIGTABLE_FEEDBACK_JOBS_TABLE;
  if (!tableId) return false;

  return withSpan(
    "bigtable.feedback_job.write",
    async span => {
      setSpanAttributes(span, {
        "db.system": "bigtable",
        "bigtable.table": tableId,
        "bigtable.operation": "mutate",
        "feedback.endpoint": params.endpoint,
      });
      const completedAt = params.completedAt ?? new Date();
      const feedbackWindowSec =
        params.endpoint === "search"
          ? config.SEARCH_FEEDBACK_MAX_AGE_SEC
          : config.FEEDBACK_MAX_AGE_SEC;
      const feedbackDeadline = new Date(
        completedAt.getTime() + feedbackWindowSec * 1000,
      );
      // The row outlives the feedback window so late feedback can be told
      // "window expired" rather than "job not found"; the deadline inside the
      // value is what enforces the window. A window configured longer than
      // the default retention still gets a row that outlives it.
      const retainUntil = new Date(
        completedAt.getTime() +
          Math.max(
            FEEDBACK_ROW_RETENTION_MS,
            feedbackWindowSec * 1000 + FEEDBACK_ROW_RETENTION_MARGIN_MS,
          ),
      );
      const value = Buffer.from(
        JSON.stringify({
          version: 1,
          requestId: params.requestId,
          teamId: params.teamId,
          refundClass:
            params.endpoint === "scrape"
              ? scrapeRefundClass(params.scrapeOptions)
              : params.endpoint,
          feedbackDeadlineMs: feedbackDeadline.getTime(),
          succeeded: params.succeeded,
          creditsBilled: params.creditsBilled,
          zeroDataRetention: params.zeroDataRetention,
          // Both describe the response, so zero data retention redacts them
          // with everything else. Feedback on a ZDR job is dropped before the
          // bounds are read, so nothing needs them for those rows.
          ...(params.zeroDataRetention
            ? {}
            : {
                ...(params.numResultsBySource
                  ? { numResultsBySource: params.numResultsBySource }
                  : {}),
                ...(params.resultCategories
                  ? { resultCategories: params.resultCategories }
                  : {}),
              }),
        }),
      );
      const table = await getBigtableTable(tableId);
      await table.mutate([
        {
          key: saltedUuidV7RowKey(params.jobId),
          method: "insert",
          data: {
            [FAMILY]: {
              [QUALIFIER]: {
                value,
                timestamp: retainUntil,
              },
            },
          },
        },
      ]);
      return true;
    },
    { zeroDataRetention: params.zeroDataRetention },
  );
}