Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { config } from "../config";
import type { ScrapeOptions } from "../controllers/v2/types";
import { includesFormat } from "./format-utils";
import { getBigtableTable } from "./bigtable-client";
import { saltedUuidV7RowKey } from "./bigtable-row-key";
import { setSpanAttributes, withSpan } from "./otel-tracer";
const FAMILY = "f";
const FEEDBACK_ROW_RETENTION_MS = 24 * 60 * 60 * 1000;
/** Headroom past a configured feedback window longer than the default retention. */
const FEEDBACK_ROW_RETENTION_MARGIN_MS = 60 * 60 * 1000;
const QUALIFIER = "v";
type FeedbackEndpoint = "search" | "scrape" | "parse" | "map";
export type RefundClass =
| "search"
| "map"
| "parse"
| "scrape_basic"
| "scrape_pdf"
| "scrape_json"
| "scrape_addon";
const REFUND_CLASSES: readonly RefundClass[] = [
"search",
"map",
"parse",
"scrape_basic",
"scrape_pdf",
"scrape_json",
"scrape_addon",
];
type FeedbackJob = {
requestId: string;
teamId: string;
refundClass: RefundClass;
feedbackDeadlineMs: number;
succeeded: boolean;
creditsBilled: number;
zeroDataRetention: boolean;
/**
* Search only. How many results each group returned, e.g.
* `{"web":3,"images":0,"news":10}`, and which vertical served each position,
* e.g. `{"web":{"1":"developer"}}`.
*
* Position feedback addresses a result as `{source, position}`, 1-indexed
* within its group, so bounding a position needs the length of that group
* and nothing else. They live here rather than beside the search row
* because this is the record the feedback window is enforced from, and it
* is the only one feedback reads.
*
* Absent on records written before these fields existed, on non-search
* jobs, and under zero data retention. Absent is not empty: `{}` means the
* search ran and every group was empty, which rejects every position.
*/
numResultsBySource?: Record<string, number>;
resultCategories?: Record<string, Record<string, string>>;
};
function parseFeedbackJob(value: Buffer | string): FeedbackJob {
const parsed: unknown = JSON.parse(value.toString());
const row = parsed as Record<string, unknown>;
if (
!parsed ||
typeof parsed !== "object" ||
row.version !== 1 ||
typeof row.requestId !== "string" ||
typeof row.teamId !== "string" ||
typeof row.refundClass !== "string" ||
!REFUND_CLASSES.includes(row.refundClass as RefundClass) ||
typeof row.feedbackDeadlineMs !== "number" ||
!Number.isFinite(row.feedbackDeadlineMs) ||
typeof row.succeeded !== "boolean" ||
typeof row.creditsBilled !== "number" ||
!Number.isFinite(row.creditsBilled) ||
typeof row.zeroDataRetention !== "boolean"
) {
throw new Error("Invalid Bigtable feedback job row");
}
// The stored shape carries a `version` for forward compatibility; the
// in-memory job does not, matching the scrape and extract state readers.
const { version: _, ...job } = row;
return job as FeedbackJob;
}
export async function readFeedbackJob(
jobId: string,
): Promise<FeedbackJob | null> {
const tableId = config.BIGTABLE_FEEDBACK_JOBS_TABLE;
if (!tableId) return null;
return withSpan("bigtable.feedback_job.read", async span => {
setSpanAttributes(span, {
"db.system": "bigtable",
"bigtable.table": tableId,
"bigtable.operation": "getRows",
});
const table = await getBigtableTable(tableId);
const [rows] = await table.getRows({
keys: [saltedUuidV7RowKey(jobId)],
filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],
});
const cells = rows[0]?.data?.[FAMILY]?.[QUALIFIER];
const cell = Array.isArray(cells) ? cells[0] : undefined;
if (cell?.value == null) {
setSpanAttributes(span, { "bigtable.read.outcome": "not_found" });
return null;
}
const job = parseFeedbackJob(cell.value);
if (job.feedbackDeadlineMs <= Date.now()) {
setSpanAttributes(span, { "bigtable.read.outcome": "expired" });
return job;
}
setSpanAttributes(span, {
"bigtable.read.outcome": "found",
"feedback.refund_class": job.refundClass,
});
return job;
});
}
function scrapeRefundClass(options: ScrapeOptions): RefundClass {
if (
options.parsers?.some(
parser =>
parser === "pdf" ||
(typeof parser === "object" && parser.type === "pdf"),
)
) {
return "scrape_pdf";
}
if (includesFormat(options.formats, "json")) return "scrape_json";
if (
includesFormat(options.formats, "screenshot") ||
(options.actions?.length ?? 0) > 0
) {
return "scrape_addon";
}
return "scrape_basic";
}
type FeedbackJobBase = {
jobId: string;
requestId: string;
teamId: string;
succeeded: boolean;
creditsBilled: number;
zeroDataRetention: boolean;
completedAt?: Date;
numResultsBySource?: Record<string, number>;
resultCategories?: Record<string, Record<string, string>>;
};
type FeedbackJobWrite = FeedbackJobBase &
(
| { endpoint: "scrape"; scrapeOptions: ScrapeOptions }
| { endpoint: Exclude<FeedbackEndpoint, "scrape"> }
);
export async function writeFeedbackJob(
params: FeedbackJobWrite,
): Promise<boolean> {
const tableId = config.BIGTABLE_FEEDBACK_JOBS_TABLE;
if (!tableId) return false;
return withSpan(
"bigtable.feedback_job.write",
async span => {
setSpanAttributes(span, {
"db.system": "bigtable",
"bigtable.table": tableId,
"bigtable.operation": "mutate",
"feedback.endpoint": params.endpoint,
});
const completedAt = params.completedAt ?? new Date();
const feedbackWindowSec =
params.endpoint === "search"
? config.SEARCH_FEEDBACK_MAX_AGE_SEC
: config.FEEDBACK_MAX_AGE_SEC;
const feedbackDeadline = new Date(
completedAt.getTime() + feedbackWindowSec * 1000,
);
// The row outlives the feedback window so late feedback can be told
// "window expired" rather than "job not found"; the deadline inside the
// value is what enforces the window. A window configured longer than
// the default retention still gets a row that outlives it.
const retainUntil = new Date(
completedAt.getTime() +
Math.max(
FEEDBACK_ROW_RETENTION_MS,
feedbackWindowSec * 1000 + FEEDBACK_ROW_RETENTION_MARGIN_MS,
),
);
const value = Buffer.from(
JSON.stringify({
version: 1,
requestId: params.requestId,
teamId: params.teamId,
refundClass:
params.endpoint === "scrape"
? scrapeRefundClass(params.scrapeOptions)
: params.endpoint,
feedbackDeadlineMs: feedbackDeadline.getTime(),
succeeded: params.succeeded,
creditsBilled: params.creditsBilled,
zeroDataRetention: params.zeroDataRetention,
// Both describe the response, so zero data retention redacts them
// with everything else. Feedback on a ZDR job is dropped before the
// bounds are read, so nothing needs them for those rows.
...(params.zeroDataRetention
? {}
: {
...(params.numResultsBySource
? { numResultsBySource: params.numResultsBySource }
: {}),
...(params.resultCategories
? { resultCategories: params.resultCategories }
: {}),
}),
}),
);
const table = await getBigtableTable(tableId);
await table.mutate([
{
key: saltedUuidV7RowKey(params.jobId),
method: "insert",
data: {
[FAMILY]: {
[QUALIFIER]: {
value,
timestamp: retainUntil,
},
},
},
},
]);
return true;
},
{ zeroDataRetention: params.zeroDataRetention },
);
}