Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/index): performance tuning
mogery7231badcf887
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 6896c5914..652904ce7 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -567,9 +567,6 @@ export async function removeJobFromGCS(jobId: string): Promise<void> {// TODO: fix the any type (we have multiple Document types in the codebase)export async function getDocFromGCS(url: string): Promise<any | null> {- // logger.info(`Getting f-engine document from GCS`, {- // url,- // });try {if (!config.GCS_FIRE_ENGINE_BUCKET_NAME) {return null;@@ -577,14 +574,18 @@ export async function getDocFromGCS(url: string): Promise<any | null> {const bucket = storage.bucket(config.GCS_FIRE_ENGINE_BUCKET_NAME);const blob = bucket.file(`${url}`);- const [exists] = await blob.exists();- if (!exists) {- return null;- }const [blobContent] = await blob.download();const parsed = JSON.parse(blobContent.toString());return parsed;} catch (error) {+ if (+ error instanceof ApiError &&+ error.code === 404 &&+ error.message.includes("No such object:")+ ) {+ return null;+ }+logger.error(`Error getting f-engine document from GCS`, {error,url,diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 0e7273559..6b6acd0d7 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -1,11 +1,9 @@+import { ApiError } from "@google-cloud/storage";import { logger } from "./logger";import { config } from "../config";import crypto from "crypto";import { storage } from "./gcs-jobs";-const credentials = config.GCS_CREDENTIALS- ? JSON.parse(atob(config.GCS_CREDENTIALS))- : undefined;const PDF_CACHE_PREFIX = "pdf-cache-v2/";/**@@ -88,14 +86,6 @@ export async function getPdfResultFromCache(const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);- const [exists] = await blob.exists();- if (!exists) {- logger.debug(`PDF RunPod result not found in GCS cache`, {- cacheKey,- });- return null;- }-const [content] = await blob.download();const result = JSON.parse(content.toString());@@ -107,6 +97,14 @@ export async function getPdfResultFromCache(...result,};} catch (error) {+ if (+ error instanceof ApiError &&+ error.code === 404 &&+ error.message.includes("No such object:")+ ) {+ return null;+ }+logger.error(`Error retrieving PDF RunPod result from GCS cache`, {error,});diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 9c957cb67..ae71536b7 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -58,10 +58,6 @@ export const index_supabase_service: SupabaseClient = new Proxy(serv, {},}) as unknown as SupabaseClient;-const credentials = config.GCS_CREDENTIALS- ? JSON.parse(atob(config.GCS_CREDENTIALS))- : undefined;-export async function getIndexFromGCS(url: string,logger?: Logger,@@ -83,8 +79,8 @@ export async function getIndexFromGCS(const [blobContent] = await blob.download();const parsed = JSON.parse(blobContent.toString());- try {- if (typeof parsed.screenshot === "string") {+ if (typeof parsed.screenshot === "string") {+ try {const screenshotUrl = new URL(parsed.screenshot);let expiresAt =parseInt(screenshotUrl.searchParams.get("Expires") ?? "0", 10) *@@ -106,26 +102,36 @@ export async function getIndexFromGCS(expiresAt < Date.now()) {logger?.info("Re-signing screenshot URL");- const [url] = await storage+ const filePath = decodeURIComponent(+ screenshotUrl.pathname.split("/")[2],+ );+ const [newUrl] = await storage.bucket(config.GCS_MEDIA_BUCKET_NAME!)- .file(decodeURIComponent(screenshotUrl.pathname.split("/")[2]))+ .file(filePath).getSignedUrl({action: "read",- expires: Date.now() + 1000 * 60 * 60 * 24 * 7, // 7 days+ expires: Date.now() + 1000 * 60 * 60 * 24 * 7,+ });+ parsed.screenshot = newUrl;++ // Persist the re-signed URL back to GCS in the background+ blob+ .save(JSON.stringify(parsed), {+ contentType: "application/json",+ })+ .catch(error => {+ logger?.warn("Error persisting re-signed screenshot URL", {+ error,+ url,+ });});- parsed.screenshot = url;-- // Update the blob- await blob.save(JSON.stringify(parsed), {- contentType: "application/json",- });}+ } catch (error) {+ logger?.warn("Error parsing screenshot URL for re-signing", {+ error,+ url,+ });}- } catch (error) {- logger?.warn("Error re-signing screenshot URL", {- error,- url,- });}setSpanAttributes(span, { "index.document_found": true });