Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: remove sentry tracing
Gergő Móricz4f8ec99637c8
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex a5ec432f1..d4bd73bbc 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -93,47 +93,36 @@ export async function scrapeHelper(let doc;- const err = await Sentry.startSpan(- {- name: "Wait for job to finish",- op: "bullmq.wait",- attributes: { job: jobId },- },- async (span) => {- try {- doc = await waitForJob(jobId, timeout);- } catch (e) {- if (- e instanceof Error &&- (e.message.startsWith("Job wait") || e.message === "timeout")- ) {- span.setAttribute("timedOut", true);- return {- success: false,- error: "Request timed out",- returnCode: 408,- };- } else if (- typeof e === "string" &&- (e.includes("Error generating completions: ") ||- e.includes("Invalid schema for function") ||- e.includes(- "LLM extraction did not match the extraction schema you provided.",- ))- ) {- return {- success: false,- error: e,- returnCode: 500,- };- } else {- throw e;- }- }- span.setAttribute("result", JSON.stringify(doc));- return null;- },- );+ try {+ doc = await waitForJob(jobId, timeout);+ } catch (e) {+ if (+ e instanceof Error &&+ (e.message.startsWith("Job wait") || e.message === "timeout")+ ) {+ return {+ success: false,+ error: "Request timed out",+ returnCode: 408,+ };+ } else if (+ typeof e === "string" &&+ (e.includes("Error generating completions: ") ||+ e.includes("Invalid schema for function") ||+ e.includes(+ "LLM extraction did not match the extraction schema you provided.",+ ))+ ) {+ return {+ success: false,+ error: e,+ returnCode: 500,+ };+ } else {+ throw e;+ }+ }+ const err = null;if (err !== null) {return err;diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 472e208eb..525db6b6c 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -75,34 +75,9 @@ export async function deepResearchController(summaries: [],});- if (Sentry.isInitialized()) {- const size = JSON.stringify(jobData).length;- await Sentry.startSpan(- {- name: "Add deep research job",- op: "queue.publish",- attributes: {- "messaging.message.id": researchId,- "messaging.destination.name": getDeepResearchQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await getDeepResearchQueue().add(researchId, {- ...jobData,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- }, { jobId: researchId });- },- );- } else {- await getDeepResearchQueue().add(researchId, jobData, {- jobId: researchId,- });- }+ await getDeepResearchQueue().add(researchId, jobData, {+ jobId: researchId,+ });return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex e6d356b85..7eba40a60 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -119,34 +119,9 @@ export async function extractController(zeroDataRetention: req.acuc?.flags?.forceZDR,});- if (Sentry.isInitialized()) {- const size = JSON.stringify(jobData).length;- await Sentry.startSpan(- {- name: "Add extract job",- op: "queue.publish",- attributes: {- "messaging.message.id": extractId,- "messaging.destination.name": getExtractQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await getExtractQueue().add(extractId, {- ...jobData,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- }, { jobId: extractId });- },- );- } else {- await getExtractQueue().add(extractId, jobData, {- jobId: extractId,- });- }+ await getExtractQueue().add(extractId, jobData, {+ jobId: extractId,+ });return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex 4177af00d..6834b819f 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -51,38 +51,9 @@ export async function generateLLMsTextController(fullText: "",});- if (Sentry.isInitialized()) {- const size = JSON.stringify(jobData).length;- await Sentry.startSpan(- {- name: "Add LLMstxt generation job",- op: "queue.publish",- attributes: {- "messaging.message.id": generationId,- "messaging.destination.name": getGenerateLlmsTxtQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await getGenerateLlmsTxtQueue().add(- generationId,- {- ...jobData,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- },- { jobId: generationId },- );- },- );- } else {- await getGenerateLlmsTxtQueue().add(generationId, jobData, {- jobId: generationId,- });- }+ await getGenerateLlmsTxtQueue().add(generationId, jobData, {+ jobId: generationId,+ });return res.status(200).json({success: true,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 768b63a96..cc1a1e4aa 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -135,35 +135,17 @@ export async function fireEngineCheckStatus(abort?: AbortSignal,production = true,): Promise<FireEngineCheckStatusSuccess> {- let status = await Sentry.startSpan(- {- name: "fire-engine: Check status",- attributes: {- jobId,- },- },- async (span) => {- const x = Date.now();- meta.logger.debug("Checking status (" + x + ")");- const res = await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,- method: "GET",- logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),- headers: {- ...(Sentry.isInitialized()- ? {- "sentry-trace": Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- }- : {}),- },- mock,- abort,- });- meta.logger.debug("Status came back (" + x + ")");- return res;- },- );+ const x = Date.now();+ meta.logger.debug("Checking status (" + x + ")");+ let status = await robustFetch({+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,+ method: "GET",+ logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),+ headers: {},+ mock,+ abort,+ });+ meta.logger.debug("Status came back (" + x + ")");// Fire-engine now saves the content to GCSif (!status.content && status.docUrl) {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.tsindex 9af6627ae..b9185a871 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts@@ -12,33 +12,16 @@ export async function fireEngineDelete(abort?: AbortSignal,production = true,) {- await Sentry.startSpan(- {- name: "fire-engine: Delete scrape",- attributes: {- jobId,- },- },- async (span) => {- await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,- method: "DELETE",- headers: {- ...(Sentry.isInitialized()- ? {- "sentry-trace": Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- }- : {}),- },- ignoreResponse: true,- ignoreFailure: true,- logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),- mock,- abort,- });- },- );+ await robustFetch({+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,+ method: "DELETE",+ headers: {},+ ignoreResponse: true,+ ignoreFailure: true,+ logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),+ mock,+ abort,+ });// We do not care whether this fails or not.}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex f3c566817..fed212dcf 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -79,34 +79,17 @@ export async function fireEngineScrape<abort?: AbortSignal,production = true,): Promise<z.infer<typeof schema>> {- const scrapeRequest = await Sentry.startSpan(- {- name: "fire-engine: Scrape",- attributes: {- url: request.url,- },- },- async (span) => {- return await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,- method: "POST",- headers: {- ...(Sentry.isInitialized()- ? {- "sentry-trace": Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- }- : {}),- },- body: request,- logger: logger.child({ method: "fireEngineScrape/robustFetch" }),- schema,- tryCount: 3,- mock,- abort,- });- },- );+ const scrapeRequest = await robustFetch({+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,+ method: "POST",+ headers: {},+ body: request,+ logger: logger.child({ method: "fireEngineScrape/robustFetch" }),+ schema,+ tryCount: 3,+ mock,+ abort,+ });return scrapeRequest;}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 2ce073f64..ef8be0349 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -261,30 +261,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =runningJobs.add(job.id);}- if (job.data && job.data.sentry && Sentry.isInitialized()) {- Sentry.continueTrace(- {- sentryTrace: job.data.sentry.trace,- baggage: job.data.sentry.baggage,- },- () => {- Sentry.startSpan(- {- name: "Index job",- attributes: {- job: job.id,- worker: process.env.FLY_MACHINE_ID ?? worker.id,- },- },- async () => {- await jobProcessor(token, job);- },- );- },- );- } else {- await jobProcessor(token, job);- }+ await jobProcessor(token, job);if (job.id) {runningJobs.delete(job.id);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 799d92d8b..d6947659a 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -157,38 +157,7 @@ export async function addScrapeJob(jobPriority: number = 10,directToBullMQ: boolean = false,): Promise<Job | null> {- if (Sentry.isInitialized()) {- const size = JSON.stringify(webScraperOptions).length;- return await Sentry.startSpan(- {- name: "Add scrape job",- op: "queue.publish",- attributes: {- "messaging.message.id": jobId,- "messaging.destination.name": getScrapeQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- return await addScrapeJobRaw(- {- ...webScraperOptions,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- },- options,- jobId,- jobPriority,- directToBullMQ,- );- },- );- } else {- return await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);- }+ return await addScrapeJobRaw(webScraperOptions, options, jobId, jobPriority, directToBullMQ);}export async function addScrapeJobs(@@ -322,33 +291,11 @@ export async function addScrapeJobs(await Promise.all(addToCQ.map(async (job) => {const size = JSON.stringify(job.data).length;- return await Sentry.startSpan(- {- name: "Add scrape job",- op: "queue.publish",- attributes: {- "messaging.message.id": job.opts.jobId,- "messaging.destination.name": getScrapeQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- const jobData = {- ...job.data,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- };-- await _addScrapeJobToConcurrencyQueue(- jobData,- job.opts,- job.opts.jobId,- job.opts.priority,- );- },+ await _addScrapeJobToConcurrencyQueue(+ job.data,+ job.opts,+ job.opts.jobId,+ job.opts.priority,);}),);@@ -356,31 +303,11 @@ export async function addScrapeJobs(await Promise.all(addToBull.map(async (job) => {const size = JSON.stringify(job.data).length;- return await Sentry.startSpan(- {- name: "Add scrape job",- op: "queue.publish",- attributes: {- "messaging.message.id": job.opts.jobId,- "messaging.destination.name": getScrapeQueue().name,- "messaging.message.body.size": size,- },- },- async (span) => {- await _addScrapeJobToBullMQ(- {- ...job.data,- sentry: {- trace: Sentry.spanToTraceHeader(span),- baggage: Sentry.spanToBaggageHeader(span),- size,- },- },- job.opts,- job.opts.jobId,- job.opts.priority,- );- },+ await _addScrapeJobToBullMQ(+ job.data,+ job.opts,+ job.opts.jobId,+ job.opts.priority,);}),);diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 53580f652..ce3abece4 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -408,68 +408,7 @@ const workerFun = async (}}- if (job.data && job.data.sentry && Sentry.isInitialized()) {- Sentry.continueTrace(- {- sentryTrace: job.data.sentry.trace,- baggage: job.data.sentry.baggage,- },- () => {- Sentry.startSpan(- {- name: "Scrape job",- attributes: {- job: job.id,- worker: process.env.FLY_MACHINE_ID ?? worker.id,- },- },- async (span) => {- await Sentry.startSpan(- {- name: "Process scrape job",- op: "queue.process",- attributes: {- "messaging.message.id": job.id,- "messaging.destination.name": scrapeQueueName,- "messaging.message.body.size": job.data.sentry.size,- "messaging.message.receive.latency":- Date.now() - (job.processedOn ?? job.timestamp),- "messaging.message.retry.count": job.attemptsMade,- },- },- async () => {- let res;- try {- res = await processJobInternal(token, job);- } finally {- await afterJobDone(job);- }-- if (res !== null) {- span.setStatus({ code: 2 }); // ERROR- } else {- span.setStatus({ code: 1 }); // OK- }- },- );- },- );- },- );- } else {- Sentry.startSpan(- {- name: "Scrape job",- attributes: {- job: job.id,- worker: process.env.FLY_MACHINE_ID ?? worker.id,- },- },- () => {- processJobInternal(token, job).finally(() => afterJobDone(job));- },- );- }+ processJobInternal(token, job).finally(() => afterJobDone(job));await sleep(gotJobInterval);} else {