Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/index): performance tuning

mogery7231badcf887
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 6896c5914..652904ce7 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -567,9 +567,6 @@ export async function removeJobFromGCS(jobId: string): Promise<void> {
// TODO: fix the any type (we have multiple Document types in the codebase)
export async function getDocFromGCS(url: string): Promise<any | null> {
- // logger.info(`Getting f-engine document from GCS`, {
- // url,
- // });
try {
if (!config.GCS_FIRE_ENGINE_BUCKET_NAME) {
return null;
@@ -577,14 +574,18 @@ export async function getDocFromGCS(url: string): Promise<any | null> {
const bucket = storage.bucket(config.GCS_FIRE_ENGINE_BUCKET_NAME);
const blob = bucket.file(`${url}`);
- const [exists] = await blob.exists();
- if (!exists) {
- return null;
- }
const [blobContent] = await blob.download();
const parsed = JSON.parse(blobContent.toString());
return parsed;
} catch (error) {
+ if (
+ error instanceof ApiError &&
+ error.code === 404 &&
+ error.message.includes("No such object:")
+ ) {
+ return null;
+ }
+
logger.error(`Error getting f-engine document from GCS`, {
error,
url,
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 0e7273559..6b6acd0d7 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -1,11 +1,9 @@
+import { ApiError } from "@google-cloud/storage";
import { logger } from "./logger";
import { config } from "../config";
import crypto from "crypto";
import { storage } from "./gcs-jobs";
-const credentials = config.GCS_CREDENTIALS
- ? JSON.parse(atob(config.GCS_CREDENTIALS))
- : undefined;
const PDF_CACHE_PREFIX = "pdf-cache-v2/";
/**
@@ -88,14 +86,6 @@ export async function getPdfResultFromCache(
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
- const [exists] = await blob.exists();
- if (!exists) {
- logger.debug(`PDF RunPod result not found in GCS cache`, {
- cacheKey,
- });
- return null;
- }
-
const [content] = await blob.download();
const result = JSON.parse(content.toString());
@@ -107,6 +97,14 @@ export async function getPdfResultFromCache(
...result,
};
} catch (error) {
+ if (
+ error instanceof ApiError &&
+ error.code === 404 &&
+ error.message.includes("No such object:")
+ ) {
+ return null;
+ }
+
logger.error(`Error retrieving PDF RunPod result from GCS cache`, {
error,
});
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 9c957cb67..ae71536b7 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -58,10 +58,6 @@ export const index_supabase_service: SupabaseClient = new Proxy(serv, {
},
}) as unknown as SupabaseClient;
-const credentials = config.GCS_CREDENTIALS
- ? JSON.parse(atob(config.GCS_CREDENTIALS))
- : undefined;
-
export async function getIndexFromGCS(
url: string,
logger?: Logger,
@@ -83,8 +79,8 @@ export async function getIndexFromGCS(
const [blobContent] = await blob.download();
const parsed = JSON.parse(blobContent.toString());
- try {
- if (typeof parsed.screenshot === "string") {
+ if (typeof parsed.screenshot === "string") {
+ try {
const screenshotUrl = new URL(parsed.screenshot);
let expiresAt =
parseInt(screenshotUrl.searchParams.get("Expires") ?? "0", 10) *
@@ -106,26 +102,36 @@ export async function getIndexFromGCS(
expiresAt < Date.now()
) {
logger?.info("Re-signing screenshot URL");
- const [url] = await storage
+ const filePath = decodeURIComponent(
+ screenshotUrl.pathname.split("/")[2],
+ );
+ const [newUrl] = await storage
.bucket(config.GCS_MEDIA_BUCKET_NAME!)
- .file(decodeURIComponent(screenshotUrl.pathname.split("/")[2]))
+ .file(filePath)
.getSignedUrl({
action: "read",
- expires: Date.now() + 1000 * 60 * 60 * 24 * 7, // 7 days
+ expires: Date.now() + 1000 * 60 * 60 * 24 * 7,
+ });
+ parsed.screenshot = newUrl;
+
+ // Persist the re-signed URL back to GCS in the background
+ blob
+ .save(JSON.stringify(parsed), {
+ contentType: "application/json",
+ })
+ .catch(error => {
+ logger?.warn("Error persisting re-signed screenshot URL", {
+ error,
+ url,
+ });
});
- parsed.screenshot = url;
-
- // Update the blob
- await blob.save(JSON.stringify(parsed), {
- contentType: "application/json",
- });
}
+ } catch (error) {
+ logger?.warn("Error parsing screenshot URL for re-signing", {
+ error,
+ url,
+ });
}
- } catch (error) {
- logger?.warn("Error re-signing screenshot URL", {
- error,
- url,
- });
}
setSpanAttributes(span, { "index.document_found": true });