Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: remove sentry tracing

Gergő Móricz4f8ec99637c8
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index a5ec432f1..d4bd73bbc 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -93,47 +93,36 @@ export async function scrapeHelper(
let doc;
- const err = await Sentry.startSpan(
- {
- name: "Wait for job to finish",
- op: "bullmq.wait",
- attributes: { job: jobId },
- },
- async (span) => {
- try {
- doc = await waitForJob(jobId, timeout);
- } catch (e) {
- if (
- e instanceof Error &&
- (e.message.startsWith("Job wait") || e.message === "timeout")
- ) {
- span.setAttribute("timedOut", true);
- return {
- success: false,
- error: "Request timed out",
- returnCode: 408,
- };
- } else if (
- typeof e === "string" &&
- (e.includes("Error generating completions: ") ||
- e.includes("Invalid schema for function") ||
- e.includes(
- "LLM extraction did not match the extraction schema you provided.",
- ))
- ) {
- return {
- success: false,
- error: e,
- returnCode: 500,
- };
- } else {
- throw e;
- }
- }
- span.setAttribute("result", JSON.stringify(doc));
- return null;
- },
- );
+ try {
+ doc = await waitForJob(jobId, timeout);
+ } catch (e) {
+ if (
+ e instanceof Error &&
+ (e.message.startsWith("Job wait") || e.message === "timeout")
+ ) {
+ return {
+ success: false,
+ error: "Request timed out",
+ returnCode: 408,
+ };
+ } else if (
+ typeof e === "string" &&
+ (e.includes("Error generating completions: ") ||
+ e.includes("Invalid schema for function") ||
+ e.includes(
+ "LLM extraction did not match the extraction schema you provided.",
+ ))
+ ) {
+ return {
+ success: false,
+ error: e,
+ returnCode: 500,
+ };
+ } else {
+ throw e;
+ }
+ }
+ const err = null;
if (err !== null) {
return err;
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts
index 472e208eb..525db6b6c 100644
--- a/apps/api/src/controllers/v1/deep-research.ts
+++ b/apps/api/src/controllers/v1/deep-research.ts
@@ -75,34 +75,9 @@ export async function deepResearchController(
summaries: [],
});
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(jobData).length;
- await Sentry.startSpan(
- {
- name: "Add deep research job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": researchId,
- "messaging.destination.name": getDeepResearchQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await getDeepResearchQueue().add(researchId, {
- ...jobData,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- }, { jobId: researchId });
- },
- );
- } else {
- await getDeepResearchQueue().add(researchId, jobData, {
- jobId: researchId,
- });
- }
+ await getDeepResearchQueue().add(researchId, jobData, {
+ jobId: researchId,
+ });
return res.status(200).json({
success: true,
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index e6d356b85..7eba40a60 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -119,34 +119,9 @@ export async function extractController(
zeroDataRetention: req.acuc?.flags?.forceZDR,
});
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(jobData).length;
- await Sentry.startSpan(
- {
- name: "Add extract job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": extractId,
- "messaging.destination.name": getExtractQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await getExtractQueue().add(extractId, {
- ...jobData,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- }, { jobId: extractId });
- },
- );
- } else {
- await getExtractQueue().add(extractId, jobData, {
- jobId: extractId,
- });
- }
+ await getExtractQueue().add(extractId, jobData, {
+ jobId: extractId,
+ });
return res.status(200).json({
success: true,
diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts
index 4177af00d..6834b819f 100644
--- a/apps/api/src/controllers/v1/generate-llmstxt.ts
+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts
@@ -51,38 +51,9 @@ export async function generateLLMsTextController(
fullText: "",
});
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(jobData).length;
- await Sentry.startSpan(
- {
- name: "Add LLMstxt generation job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": generationId,
- "messaging.destination.name": getGenerateLlmsTxtQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await getGenerateLlmsTxtQueue().add(
- generationId,
- {
- ...jobData,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- },
- { jobId: generationId },
- );
- },
- );
- } else {
- await getGenerateLlmsTxtQueue().add(generationId, jobData, {
- jobId: generationId,
- });
- }
+ await getGenerateLlmsTxtQueue().add(generationId, jobData, {
+ jobId: generationId,
+ });
return res.status(200).json({
success: true,
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 768b63a96..cc1a1e4aa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -135,35 +135,17 @@ export async function fireEngineCheckStatus(
abort?: AbortSignal,
production = true,
): Promise<FireEngineCheckStatusSuccess> {
- let status = await Sentry.startSpan(
- {
- name: "fire-engine: Check status",
- attributes: {
- jobId,
- },
- },
- async (span) => {
- const x = Date.now();
- meta.logger.debug("Checking status (" + x + ")");
- const res = await robustFetch({
- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
- method: "GET",
- logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),
- headers: {
- ...(Sentry.isInitialized()
- ? {
- "sentry-trace": Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- }
- : {}),
- },
- mock,
- abort,
- });
- meta.logger.debug("Status came back (" + x + ")");
- return res;
- },
- );
+ const x = Date.now();
+ meta.logger.debug("Checking status (" + x + ")");
+ let status = await robustFetch({
+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
+ method: "GET",
+ logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),
+ headers: {},
+ mock,
+ abort,
+ });
+ meta.logger.debug("Status came back (" + x + ")");
// Fire-engine now saves the content to GCS
if (!status.content && status.docUrl) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
index 9af6627ae..b9185a871 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts
@@ -12,33 +12,16 @@ export async function fireEngineDelete(
abort?: AbortSignal,
production = true,
) {
- await Sentry.startSpan(
- {
- name: "fire-engine: Delete scrape",
- attributes: {
- jobId,
- },
- },
- async (span) => {
- await robustFetch({
- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
- method: "DELETE",
- headers: {
- ...(Sentry.isInitialized()
- ? {
- "sentry-trace": Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- }
- : {}),
- },
- ignoreResponse: true,
- ignoreFailure: true,
- logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),
- mock,
- abort,
- });
- },
- );
+ await robustFetch({
+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
+ method: "DELETE",
+ headers: {},
+ ignoreResponse: true,
+ ignoreFailure: true,
+ logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),
+ mock,
+ abort,
+ });
// We do not care whether this fails or not.
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index f3c566817..fed212dcf 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -79,34 +79,17 @@ export async function fireEngineScrape<
abort?: AbortSignal,
production = true,
): Promise<z.infer<typeof schema>> {
- const scrapeRequest = await Sentry.startSpan(
- {
- name: "fire-engine: Scrape",
- attributes: {
- url: request.url,
- },
- },
- async (span) => {
- return await robustFetch({
- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,
- method: "POST",
- headers: {
- ...(Sentry.isInitialized()
- ? {
- "sentry-trace": Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- }
- : {}),
- },
- body: request,
- logger: logger.child({ method: "fireEngineScrape/robustFetch" }),
- schema,
- tryCount: 3,
- mock,
- abort,
- });
- },
- );
+ const scrapeRequest = await robustFetch({
+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,
+ method: "POST",
+ headers: {},
+ body: request,
+ logger: logger.child({ method: "fireEngineScrape/robustFetch" }),
+ schema,
+ tryCount: 3,
+ mock,
+ abort,
+ });
return scrapeRequest;
}
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 2ce073f64..ef8be0349 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -261,30 +261,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =
runningJobs.add(job.id);
}
- if (job.data && job.data.sentry && Sentry.isInitialized()) {
- Sentry.continueTrace(
- {
- sentryTrace: job.data.sentry.trace,
- baggage: job.data.sentry.baggage,
- },
- () => {
- Sentry.startSpan(
- {
- name: "Index job",
- attributes: {
- job: job.id,
- worker: process.env.FLY_MACHINE_ID ?? worker.id,
- },
- },
- async () => {
- await jobProcessor(token, job);
- },
- );
- },
- );
- } else {
- await jobProcessor(token, job);
- }
+ await jobProcessor(token, job);
if (job.id) {
runningJobs.delete(job.id);
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 799d92d8b..d6947659a 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -157,38 +157,7 @@ export async function addScrapeJob(
jobPriority: number = 10,
directToBullMQ: boolean = false,
): Promise<Job | null> {
- if (Sentry.isInitialized()) {
- const size = JSON.stringify(webScraperOptions).length;
- return await Sentry.startSpan(
- {
- name: "Add scrape job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": jobId,
- "messaging.destination.name": getScrapeQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- return await addScrapeJobRaw(
- {
- ...webScraperOptions,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- },
- options,
- jobId,
- jobPriority,
- directToBullMQ,
- );
- },
- );
- } else {
- return await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);
- }
+ return await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);
}
export async function addScrapeJobs(
@@ -322,33 +291,11 @@ export async function addScrapeJobs(
await Promise.all(
addToCQ.map(async (job) => {
const size = JSON.stringify(job.data).length;
- return await Sentry.startSpan(
- {
- name: "Add scrape job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": job.opts.jobId,
- "messaging.destination.name": getScrapeQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- const jobData = {
- ...job.data,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- };
-
- await _addScrapeJobToConcurrencyQueue(
- jobData,
- job.opts,
- job.opts.jobId,
- job.opts.priority,
- );
- },
+ await _addScrapeJobToConcurrencyQueue(
+ job.data,
+ job.opts,
+ job.opts.jobId,
+ job.opts.priority,
);
}),
);
@@ -356,31 +303,11 @@ export async function addScrapeJobs(
await Promise.all(
addToBull.map(async (job) => {
const size = JSON.stringify(job.data).length;
- return await Sentry.startSpan(
- {
- name: "Add scrape job",
- op: "queue.publish",
- attributes: {
- "messaging.message.id": job.opts.jobId,
- "messaging.destination.name": getScrapeQueue().name,
- "messaging.message.body.size": size,
- },
- },
- async (span) => {
- await _addScrapeJobToBullMQ(
- {
- ...job.data,
- sentry: {
- trace: Sentry.spanToTraceHeader(span),
- baggage: Sentry.spanToBaggageHeader(span),
- size,
- },
- },
- job.opts,
- job.opts.jobId,
- job.opts.priority,
- );
- },
+ await _addScrapeJobToBullMQ(
+ job.data,
+ job.opts,
+ job.opts.jobId,
+ job.opts.priority,
);
}),
);
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 53580f652..ce3abece4 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -408,68 +408,7 @@ const workerFun = async (
}
}
- if (job.data && job.data.sentry && Sentry.isInitialized()) {
- Sentry.continueTrace(
- {
- sentryTrace: job.data.sentry.trace,
- baggage: job.data.sentry.baggage,
- },
- () => {
- Sentry.startSpan(
- {
- name: "Scrape job",
- attributes: {
- job: job.id,
- worker: process.env.FLY_MACHINE_ID ?? worker.id,
- },
- },
- async (span) => {
- await Sentry.startSpan(
- {
- name: "Process scrape job",
- op: "queue.process",
- attributes: {
- "messaging.message.id": job.id,
- "messaging.destination.name": scrapeQueueName,
- "messaging.message.body.size": job.data.sentry.size,
- "messaging.message.receive.latency":
- Date.now() - (job.processedOn ?? job.timestamp),
- "messaging.message.retry.count": job.attemptsMade,
- },
- },
- async () => {
- let res;
- try {
- res = await processJobInternal(token, job);
- } finally {
- await afterJobDone(job);
- }
-
- if (res !== null) {
- span.setStatus({ code: 2 }); // ERROR
- } else {
- span.setStatus({ code: 1 }); // OK
- }
- },
- );
- },
- );
- },
- );
- } else {
- Sentry.startSpan(
- {
- name: "Scrape job",
- attributes: {
- job: job.id,
- worker: process.env.FLY_MACHINE_ID ?? worker.id,
- },
- },
- () => {
- processJobInternal(token, job).finally(() => afterJobDone(job));
- },
- );
- }
+ processJobInternal(token, job).finally(() => afterJobDone(job));
await sleep(gotJobInterval);
} else {