Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { and, eq } from "drizzle-orm";
import { DrizzleQueryError } from "drizzle-orm/errors";
import { config } from "../../../config";
import { db, dbRr } from "../../../db/connection";
import * as schema from "../../../db/schema";
import {
readFeedbackJob,
type RefundClass,
} from "../../../lib/feedback-job-store";
import { logger } from "../../../lib/logger";
import { EndpointFeedbackEndpoint } from "../types";
import type { SearchResultType } from "../../../lib/entities";
import {
FeedbackJobRow,
FeedbackRecordOptions,
RefundPolicySnapshot,
ValuableResultInput,
} from "./internal-types";
type DbError = { code?: string } & Record<string, unknown>;
/**
* Drizzle wraps a failed query in DrizzleQueryError and keeps the PostgreSQL
* error, with its SQLSTATE `code`, as the cause. Callers branch on that code,
* for example a unique violation on a second submission for the same job.
*/
function dbError(error: unknown): DbError {
return (error instanceof DrizzleQueryError ? error.cause : error) as DbError;
}
type ExistingFeedback = {
id: string;
credits_refunded: number | null;
};
type ValuableResultDocument = {
documentId: string;
searchId: string;
/** The `data` group the client addressed, as sent: web | images | news. */
requestedSource: SearchResultType;
/** Analytics spelling, as used in `documentId`. */
resultType: SearchResultDocumentType;
resultIndex: number;
position: number;
/**
* Which vertical served this result, as the response tagged it — `developer`
* for an index-served hit, `github` / `research` / `pdf` for a URL-derived
* one. Absent when the result carried no category, or when the feedback job
* predates `resultCategories`. This is the attribution half of the label:
* `requestedSource` says where in `data` the result sat, `category` says who
* answered.
*/
category?: string;
reason?: string;
source: "position";
};
/**
* Result type as recorded in analytics. The API groups image results under
* `images` (matching `data.images`), but the ClickHouse `search_results` table
* writes the singular `image`; document IDs use the analytics spelling so they
* join directly on (search_id, result_type, result_index).
*/
type SearchResultDocumentType = "web" | "news" | "image";
const RESULT_DOCUMENT_TYPES: Record<
SearchResultType,
SearchResultDocumentType
> = {
web: "web",
news: "news",
images: "image",
};
function searchFeedbackResultDocumentId(
searchId: string,
resultType: SearchResultDocumentType,
resultIndex: number,
): string {
return `search:${searchId}:${resultType}:${resultIndex}`;
}
/**
* Number of results each `data` group returned, as the feedback job recorded
* it. Returns null when the job predates the field or carries an unrecognised
* shape — which is not the same as an empty object, meaning the search ran and
* every group was empty.
*/
function resultCountsBySource(
job: FeedbackJobRow,
): Partial<Record<SearchResultType, number>> | null {
const counts = job.num_results_by_source;
if (counts === null || counts === undefined || typeof counts !== "object") {
return null;
}
// An object with no usable entries still counts as known: the search ran
// and no group this API can address returned anything, which rejects every
// position. Only a missing or non-object value is unknown.
const entries = Object.entries(counts as Record<string, unknown>).filter(
(entry): entry is [SearchResultType, number] =>
entry[0] in RESULT_DOCUMENT_TYPES &&
typeof entry[1] === "number" &&
Number.isInteger(entry[1]) &&
entry[1] >= 0,
);
return Object.fromEntries(entries);
}
/**
* Vertical that served each position, as the feedback job recorded it. Returns
* null when the job predates the field or carries an unrecognised shape, which
* again is not the same as an empty object.
*/
function resultCategoriesBySource(
job: FeedbackJobRow,
): Partial<Record<SearchResultType, Record<string, string>>> | null {
const categories = job.result_categories;
if (
categories === null ||
categories === undefined ||
typeof categories !== "object"
) {
return null;
}
const bySource: Partial<Record<SearchResultType, Record<string, string>>> =
{};
for (const [source, positions] of Object.entries(
categories as Record<string, unknown>,
)) {
if (!(source in RESULT_DOCUMENT_TYPES)) continue;
if (!positions || typeof positions !== "object") continue;
const byPosition: Record<string, string> = {};
for (const [position, category] of Object.entries(
positions as Record<string, unknown>,
)) {
if (typeof category === "string" && category.length > 0) {
byPosition[position] = category;
}
}
bySource[source as SearchResultType] = byPosition;
}
return bySource;
}
/**
* Highest 1-indexed position addressable in `data[source]`, or null when the
* job carries no usable bound.
*
* The per-source counts bound every group exactly, so there is no looser
* fallback to reach for: a group the counts do not mention returned nothing
* and has no valid position. Null means the job predates the field — feedback
* is only accepted within SEARCH_FEEDBACK_MAX_AGE_SEC of the search, so that
* only covers searches run in the couple of minutes spanning the deploy, and
* closes on its own.
*/
function maxResultPosition(
job: FeedbackJobRow,
source: SearchResultType,
): number | null {
const counts = resultCountsBySource(job);
if (counts === null) return null;
return counts[source] ?? 0;
}
function valuableResultDocuments(
job: FeedbackJobRow,
results: ValuableResultInput[] | undefined,
): ValuableResultDocument[] {
if (job.endpoint !== "search" || !results?.length) return [];
// Rejects hallucinated positions, which would otherwise be stored as
// false-positive relevance labels. Zero is a real bound: a group that
// returned nothing has no valuable positions.
const maxPositions = new Map<SearchResultType, number | null>();
const seen = new Set<string>();
const categories = resultCategoriesBySource(job);
return results.flatMap(({ source, position, reason }) => {
if (!(source in RESULT_DOCUMENT_TYPES)) return [];
if (!Number.isInteger(position) || position <= 0) return [];
if (!maxPositions.has(source)) {
maxPositions.set(source, maxResultPosition(job, source));
}
const maxPosition = maxPositions.get(source) ?? null;
if (maxPosition !== null && position > maxPosition) return [];
const key = `${source}:${position}`;
if (seen.has(key)) return [];
seen.add(key);
const resultType = RESULT_DOCUMENT_TYPES[source];
const resultIndex = position - 1;
const category = categories?.[source]?.[String(position)];
return [
{
documentId: searchFeedbackResultDocumentId(
job.id,
resultType,
resultIndex,
),
searchId: job.id,
requestedSource: source,
resultType,
resultIndex,
position,
...(category ? { category } : {}),
...(reason ? { reason } : {}),
source: "position" as const,
},
];
});
}
function feedbackMetadata(
options: FeedbackRecordOptions,
valuableResultDocs: ValuableResultDocument[],
): Record<string, unknown> {
return {
...(options.feedback.metadata ?? {}),
...(options.feedback.url ? { url: options.feedback.url } : {}),
...(options.feedback.pageNumbers
? { pageNumbers: options.feedback.pageNumbers }
: {}),
...(valuableResultDocs.length > 0
? {
// Echoed back in the vocabulary the client sent, not the analytics
// spelling used in the document IDs.
valuableResults: valuableResultDocs.map(doc => ({
source: doc.requestedSource,
position: doc.position,
...(doc.category ? { category: doc.category } : {}),
})),
valuableResultDocumentIds: valuableResultDocs.map(
doc => doc.documentId,
),
}
: {}),
};
}
export async function lookupFeedbackJob(
endpoint: EndpointFeedbackEndpoint,
jobId: string,
dbTeamId: string,
): Promise<FeedbackJobRow | null> {
let job;
try {
job = await readFeedbackJob(jobId);
} catch (error) {
logger.warn("Bigtable feedback job read failed", {
error,
jobId,
endpoint,
});
return null;
}
if (!job) return null;
const storedEndpoint = endpointForRefundClass(job.refundClass);
if (job.teamId !== dbTeamId || storedEndpoint !== endpoint) return null;
const feedbackWindowSec =
endpoint === "search"
? config.SEARCH_FEEDBACK_MAX_AGE_SEC
: config.FEEDBACK_MAX_AGE_SEC;
return {
endpoint,
id: jobId,
request_id: job.requestId,
team_id: job.teamId,
credits_cost: job.creditsBilled,
created_at: new Date(
job.feedbackDeadlineMs - feedbackWindowSec * 1000,
).toISOString(),
is_successful: job.succeeded,
options: null,
feedback_deadline_ms: job.feedbackDeadlineMs,
refund_class: job.refundClass,
zero_data_retention: job.zeroDataRetention,
num_results_by_source:
endpoint === "search" ? (job.numResultsBySource ?? null) : null,
result_categories:
endpoint === "search" ? (job.resultCategories ?? null) : null,
};
}
function endpointForRefundClass(
refundClass: RefundClass,
): EndpointFeedbackEndpoint {
switch (refundClass) {
case "search":
case "map":
case "parse":
return refundClass;
default:
return "scrape";
}
}
export async function insertFeedback(params: {
feedbackId: string;
options: FeedbackRecordOptions;
job: FeedbackJobRow;
dbTeamId: string;
apiKeyId?: number | null;
}): Promise<DbError | null> {
const { feedbackId, options, job, dbTeamId, apiKeyId } = params;
const valuableResultDocs = valuableResultDocuments(
job,
options.feedback.valuableResults,
);
try {
await db.insert(schema.search_feedback).values({
id: feedbackId,
search_id: options.endpoint === "search" ? options.jobId : null,
endpoint: options.endpoint,
job_id: options.jobId,
request_id: job.request_id,
api_version: "v2",
team_id: dbTeamId,
api_key_id: apiKeyId ?? null,
overall_rating: options.feedback.rating,
issue_types: options.feedback.issues ?? [],
tags: options.feedback.tags ?? [],
comment: options.feedback.note ?? null,
valuable_sources: [
...(options.feedback.valuableSources ?? []),
...valuableResultDocs,
],
missing_content: options.feedback.missingContent ?? [],
query_suggestions: options.feedback.querySuggestions ?? null,
metadata: feedbackMetadata(options, valuableResultDocs),
job_status: job.is_successful === false ? "failed" : "completed",
credits_billed: job.credits_cost ?? 0,
credits_refunded: 0,
refund_policy: null,
integration: options.feedback.integration ?? null,
origin: options.feedback.origin ?? null,
});
return null;
} catch (error) {
return dbError(error);
}
}
async function findFeedbackByJob(
dbTeamId: string,
endpoint: EndpointFeedbackEndpoint,
jobId: string,
): Promise<ExistingFeedback | null> {
const [row] = await dbRr
.select({
id: schema.search_feedback.id,
credits_refunded: schema.search_feedback.credits_refunded,
})
.from(schema.search_feedback)
.where(
and(
eq(schema.search_feedback.team_id, dbTeamId),
eq(schema.search_feedback.endpoint, endpoint),
eq(schema.search_feedback.job_id, jobId),
),
)
.limit(1);
return row ?? null;
}
async function findSearchFeedbackByLegacyId(
dbTeamId: string,
searchId: string,
): Promise<ExistingFeedback | null> {
const [row] = await dbRr
.select({
id: schema.search_feedback.id,
credits_refunded: schema.search_feedback.credits_refunded,
})
.from(schema.search_feedback)
.where(
and(
eq(schema.search_feedback.team_id, dbTeamId),
eq(schema.search_feedback.search_id, searchId),
),
)
.limit(1);
return row ?? null;
}
export async function findExistingFeedback(
dbTeamId: string,
endpoint: EndpointFeedbackEndpoint,
jobId: string,
): Promise<ExistingFeedback | null> {
return (
(await findFeedbackByJob(dbTeamId, endpoint, jobId)) ??
(endpoint === "search"
? await findSearchFeedbackByLegacyId(dbTeamId, jobId)
: null)
);
}
export async function updateFeedbackRefundDetails(
feedbackId: string,
creditsRefunded: number,
policy: RefundPolicySnapshot,
): Promise<DbError | null> {
try {
await db
.update(schema.search_feedback)
.set({
credits_refunded: creditsRefunded,
refund_policy: policy,
updated_at: new Date().toISOString(),
})
.where(eq(schema.search_feedback.id, feedbackId));
return null;
} catch (error) {
return dbError(error);
}
}