Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): stop using bulljobs_teams table (#1962)
Gergő Móricz5e64ecba0705
diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.tsindex ff8b88040..3d5249b9c 100644--- a/apps/api/src/controllers/v0/crawl-cancel.ts+++ b/apps/api/src/controllers/v0/crawl-cancel.ts@@ -1,7 +1,6 @@import { Request, Response } from "express";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";-import { supabase_service } from "../../../src/services/supabase";import { logger } from "../../../src/lib/logger";import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";import * as Sentry from "@sentry/node";@@ -11,8 +10,6 @@ configDotenv();export async function crawlCancelController(req: Request, res: Response) {try {- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";-const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);if (!auth.success) {return res.status(auth.status).json({ error: auth.error });@@ -36,19 +33,8 @@ export async function crawlCancelController(req: Request, res: Response) {}// check if the job belongs to the team- if (useDbAuthentication) {- const { data, error: supaError } = await supabase_service- .from("bulljobs_teams")- .select("*")- .eq("job_id", req.params.jobId)- .eq("team_id", team_id);- if (supaError) {- return res.status(500).json({ error: supaError.message });- }-- if (data.length === 0) {- return res.status(403).json({ error: "Unauthorized" });- }+ if (sc.team_id !== team_id) {+ return res.status(403).json({ error: "Unauthorized" });}try {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 63b90e809..65e5c4ed1 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -4,7 +4,6 @@ import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";import { addScrapeJob } from "../../../src/services/queue-jobs";import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";-import { logCrawl } from "../../../src/services/logging/crawl_log";import { validateIdempotencyKey } from "../../../src/services/idempotency/validate";import { createIdempotencyKey } from "../../../src/services/idempotency/create";import {@@ -159,8 +158,6 @@ export async function crawlController(req: Request, res: Response) {// }// }- await logCrawl(id, team_id);-const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(pageOptions,undefined,diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 9a034c150..57f6c4499 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -17,7 +17,6 @@ import {saveCrawl,StoredCrawl,} from "../../lib/crawl-redis";-import { logCrawl } from "../../services/logging/crawl_log";import { getJobPriority } from "../../lib/job-priority";import { addScrapeJobs } from "../../services/queue-jobs";import { callWebhook } from "../../services/webhook";@@ -96,10 +95,6 @@ export async function batchScrapeController(account: req.account,});- if (!req.body.appendToId) {- await logCrawl(id, req.auth.team_id);- }-const sc: StoredCrawl = req.body.appendToId? ((await getCrawl(req.body.appendToId)) as StoredCrawl): {diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.tsindex 00af8b31f..8ddd9fdba 100644--- a/apps/api/src/controllers/v1/crawl-cancel.ts+++ b/apps/api/src/controllers/v1/crawl-cancel.ts@@ -1,5 +1,4 @@import { Response } from "express";-import { supabase_service } from "../../services/supabase";import { logger } from "../../lib/logger";import { getCrawl, saveCrawl } from "../../lib/crawl-redis";import * as Sentry from "@sentry/node";@@ -12,27 +11,13 @@ export async function crawlCancelController(res: Response,) {try {- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";-const sc = await getCrawl(req.params.jobId);if (!sc) {return res.status(404).json({ error: "Job not found" });}- // check if the job belongs to the team- if (useDbAuthentication) {- const { data, error: supaError } = await supabase_service- .from("bulljobs_teams")- .select("*")- .eq("job_id", req.params.jobId)- .eq("team_id", req.auth.team_id);- if (supaError) {- return res.status(500).json({ error: supaError.message });- }-- if (data.length === 0) {- return res.status(403).json({ error: "Unauthorized" });- }+ if (sc.team_id !== req.auth.team_id) {+ return res.status(403).json({ error: "Unauthorized" });}try {diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 1f75bc0ab..ab5d2decd 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -8,7 +8,6 @@ import {toLegacyCrawlerOptions,} from "./types";import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";-import { logCrawl } from "../../services/logging/crawl_log";import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";@@ -43,8 +42,6 @@ export async function crawlController(account: req.account,});- await logCrawl(id, req.auth.team_id);-let { remainingCredits } = req.account!;const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";if (!useDbAuthentication) {diff --git a/apps/api/src/services/logging/crawl_log.ts b/apps/api/src/services/logging/crawl_log.tsdeleted file mode 100644index 86f885293..000000000--- a/apps/api/src/services/logging/crawl_log.ts+++ /dev/null@@ -1,22 +0,0 @@-import { supabase_service } from "../supabase";-import { logger } from "../../../src/lib/logger";-import { configDotenv } from "dotenv";-configDotenv();--export async function logCrawl(job_id: string, team_id: string) {- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (useDbAuthentication) {- try {- const { data, error } = await supabase_service- .from("bulljobs_teams")- .insert([- {- job_id: job_id,- team_id: team_id,- },- ]);- } catch (error) {- logger.error(`Error logging crawl job to supabase:\n${error}`);- }- }-}