Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): stop using bulljobs_teams table (#1962)

Gergő Móricz5e64ecba0705
diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts
index ff8b88040..3d5249b9c 100644
--- a/apps/api/src/controllers/v0/crawl-cancel.ts
+++ b/apps/api/src/controllers/v0/crawl-cancel.ts
@@ -1,7 +1,6 @@
import { Request, Response } from "express";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
-import { supabase_service } from "../../../src/services/supabase";
import { logger } from "../../../src/lib/logger";
import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";
import * as Sentry from "@sentry/node";
@@ -11,8 +10,6 @@ configDotenv();
export async function crawlCancelController(req: Request, res: Response) {
try {
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
-
const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);
if (!auth.success) {
return res.status(auth.status).json({ error: auth.error });
@@ -36,19 +33,8 @@ export async function crawlCancelController(req: Request, res: Response) {
}
// check if the job belongs to the team
- if (useDbAuthentication) {
- const { data, error: supaError } = await supabase_service
- .from("bulljobs_teams")
- .select("*")
- .eq("job_id", req.params.jobId)
- .eq("team_id", team_id);
- if (supaError) {
- return res.status(500).json({ error: supaError.message });
- }
-
- if (data.length === 0) {
- return res.status(403).json({ error: "Unauthorized" });
- }
+ if (sc.team_id !== team_id) {
+ return res.status(403).json({ error: "Unauthorized" });
}
try {
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 63b90e809..65e5c4ed1 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -4,7 +4,6 @@ import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
import { addScrapeJob } from "../../../src/services/queue-jobs";
import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
-import { logCrawl } from "../../../src/services/logging/crawl_log";
import { validateIdempotencyKey } from "../../../src/services/idempotency/validate";
import { createIdempotencyKey } from "../../../src/services/idempotency/create";
import {
@@ -159,8 +158,6 @@ export async function crawlController(req: Request, res: Response) {
// }
// }
- await logCrawl(id, team_id);
-
const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(
pageOptions,
undefined,
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 9a034c150..57f6c4499 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -17,7 +17,6 @@ import {
saveCrawl,
StoredCrawl,
} from "../../lib/crawl-redis";
-import { logCrawl } from "../../services/logging/crawl_log";
import { getJobPriority } from "../../lib/job-priority";
import { addScrapeJobs } from "../../services/queue-jobs";
import { callWebhook } from "../../services/webhook";
@@ -96,10 +95,6 @@ export async function batchScrapeController(
account: req.account,
});
- if (!req.body.appendToId) {
- await logCrawl(id, req.auth.team_id);
- }
-
const sc: StoredCrawl = req.body.appendToId
? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
: {
diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts
index 00af8b31f..8ddd9fdba 100644
--- a/apps/api/src/controllers/v1/crawl-cancel.ts
+++ b/apps/api/src/controllers/v1/crawl-cancel.ts
@@ -1,5 +1,4 @@
import { Response } from "express";
-import { supabase_service } from "../../services/supabase";
import { logger } from "../../lib/logger";
import { getCrawl, saveCrawl } from "../../lib/crawl-redis";
import * as Sentry from "@sentry/node";
@@ -12,27 +11,13 @@ export async function crawlCancelController(
res: Response,
) {
try {
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
-
const sc = await getCrawl(req.params.jobId);
if (!sc) {
return res.status(404).json({ error: "Job not found" });
}
- // check if the job belongs to the team
- if (useDbAuthentication) {
- const { data, error: supaError } = await supabase_service
- .from("bulljobs_teams")
- .select("*")
- .eq("job_id", req.params.jobId)
- .eq("team_id", req.auth.team_id);
- if (supaError) {
- return res.status(500).json({ error: supaError.message });
- }
-
- if (data.length === 0) {
- return res.status(403).json({ error: "Unauthorized" });
- }
+ if (sc.team_id !== req.auth.team_id) {
+ return res.status(403).json({ error: "Unauthorized" });
}
try {
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 1f75bc0ab..ab5d2decd 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -8,7 +8,6 @@ import {
toLegacyCrawlerOptions,
} from "./types";
import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
-import { logCrawl } from "../../services/logging/crawl_log";
import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
@@ -43,8 +42,6 @@ export async function crawlController(
account: req.account,
});
- await logCrawl(id, req.auth.team_id);
-
let { remainingCredits } = req.account!;
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
if (!useDbAuthentication) {
diff --git a/apps/api/src/services/logging/crawl_log.ts b/apps/api/src/services/logging/crawl_log.ts
deleted file mode 100644
index 86f885293..000000000
--- a/apps/api/src/services/logging/crawl_log.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { supabase_service } from "../supabase";
-import { logger } from "../../../src/lib/logger";
-import { configDotenv } from "dotenv";
-configDotenv();
-
-export async function logCrawl(job_id: string, team_id: string) {
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
- if (useDbAuthentication) {
- try {
- const { data, error } = await supabase_service
- .from("bulljobs_teams")
- .insert([
- {
- job_id: job_id,
- team_id: team_id,
- },
- ]);
- } catch (error) {
- logger.error(`Error logging crawl job to supabase:\n${error}`);
- }
- }
-}