Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: repalce ts-prune with knip (#2148)

tomd7541885a85e
diff --git a/apps/api/.husky/pre-commit b/apps/api/.husky/pre-commit
index 5f8a22160..629a923ce 100644
--- a/apps/api/.husky/pre-commit
+++ b/apps/api/.husky/pre-commit
@@ -1 +1 @@
-cd apps/api && pnpm lint-staged
\ No newline at end of file
+cd apps/api && pnpm knip --cache && pnpm lint-staged
\ No newline at end of file
diff --git a/apps/api/knip.config.ts b/apps/api/knip.config.ts
new file mode 100644
index 000000000..09a07399e
--- /dev/null
+++ b/apps/api/knip.config.ts
@@ -0,0 +1,13 @@
+import type { KnipConfig } from "knip";
+
+const config: KnipConfig = {
+ workspaces: {
+ ".": {
+ entry: ["src/services/worker/**/*.ts", "src/services/**/*-worker.ts"],
+ project: ["src/**/*.ts"],
+ },
+ },
+ ignore: ["native/**"],
+};
+
+export default config;
diff --git a/apps/api/package.json b/apps/api/package.json
index ab914b2b4..fc294a1d7 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -2,7 +2,7 @@
"name": "firecrawl-scraper-js",
"version": "1.0.0",
"description": "",
- "main": "index.js",
+ "main": "src/index.ts",
"scripts": {
"start": "tsc && node dist/src/harness.js --start-built",
"dev": "tsx src/harness.ts --start",
@@ -30,7 +30,8 @@
"mongo-docker-console": "docker exec -it mongodb mongosh",
"run-example": "npx ts-node src/example.ts",
"sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist",
- "prepare": "cd ../.. && husky ./apps/api/.husky"
+ "prepare": "cd ../.. && husky ./apps/api/.husky",
+ "knip": "knip"
},
"author": "",
"license": "ISC",
@@ -55,7 +56,6 @@
"supertest": "^6.3.3",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
- "ts-prune": "^0.10.3",
"tsc-watch": "^7.1.1",
"tsx": "^4.20.3",
"typescript": "^5.8.3"
@@ -115,7 +115,6 @@
"mammoth": "^1.7.2",
"marked": "^14.1.2",
"ollama-ai-provider": "^1.2.0",
- "openai": "^5.20.0",
"parse-diff": "^0.11.1",
"pdf-parse": "^1.1.1",
"pg": "^8.16.3",
@@ -161,8 +160,7 @@
}
},
"lint-staged": {
- "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write",
- "*.{js,tsx,ts,tsx}": "ts-prune --skip \"native/\" --error"
+ "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write"
},
"packageManager": "[email protected]"
}
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index ea638a8ab..679108edd 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -174,9 +174,6 @@ importers:
ollama-ai-provider:
specifier: ^1.2.0
version: 1.2.0([email protected])
- openai:
- specifier: ^5.20.0
parse-diff:
specifier: ^0.11.1
version: 0.11.1
@@ -304,9 +301,6 @@ importers:
ts-node:
specifier: ^10.9.1
version: 10.9.2(@types/[email protected])([email protected])
- ts-prune:
- specifier: ^0.10.3
- version: 0.10.3
tsc-watch:
specifier: ^7.1.1
version: 7.1.1([email protected])
@@ -3103,9 +3097,6 @@ packages:
resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}
engines: {node: '>= 10'}
- '@ts-morph/[email protected]':
- resolution: {integrity: sha512-4tUmeLyXJnJWvTFOKtcNJ1yh0a3SsTLi2MUoyj8iUNznFRN1ZquaNe7Oukqrnki2FzZkm0J9adCNLDZxUzvj+w==}
-
'@tsconfig/[email protected]':
resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}
@@ -3223,9 +3214,6 @@ packages:
resolution: {integrity: sha512-kK1eBS/Adeyis+3OlBDMeQQuasIDLUYXsi2T15ccNJ0iyUpQ4xDF7svFu3+bGVrI0CMBUclPciz+lsQR3JX3TQ==}
- '@types/[email protected]':
- resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==}
-
resolution: {integrity: sha512-+gbBHbNCVGGYw1S9lAIIvrHW47UYOhMIFUsJcMkMrzy1Jf0vulBN3XQIjPgnoOXveMuHnF3b57fXROnY/Or7eg==}
@@ -3820,9 +3808,6 @@ packages:
resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}
engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}
- resolution: {integrity: sha512-NiujjUFB4SwScJq2bwbYUtXbZhBSlY6vYzm++3Q6oC+U+injTqfPYFK8wS9COOmb2lueqp0ZRB4nK1VYeHgNyw==}
-
resolution: {integrity: sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==}
@@ -3870,10 +3855,6 @@ packages:
resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==}
- resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==}
- engines: {node: '>= 6'}
-
resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}
@@ -3914,10 +3895,6 @@ packages:
resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==}
engines: {node: '>= 0.10'}
- resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==}
- engines: {node: '>=10'}
-
resolution: {integrity: sha512-nPklKJ5qtmY5MdBKw1NiBAoyx5Sa7p2yPpljZyQ7gyCN1m+eMFs9I6CT37Mxt8zvR5L3VzD3DJBE4WQzX3WF4A==}
@@ -4684,10 +4661,6 @@ packages:
resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}
- resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==}
- engines: {node: '>=6'}
-
resolution: {integrity: sha512-5tCuY9BV8ujfOpwtAGgsTx9CGUapcFMEEyByLv1B+v2+6DhAcw+Zr0nhQT7uwaZ7DiourxFEscghOR8e1aPLQw==}
@@ -5366,11 +5339,6 @@ packages:
typescript:
optional: true
- resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}
- engines: {node: '>=10'}
- hasBin: true
-
resolution: {integrity: sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==}
@@ -5545,18 +5513,6 @@ packages:
resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==}
engines: {node: '>=18'}
- resolution: {integrity: sha512-Bmc2zLM/YWgFrDpXr9hwXqGGDdMmMpE9+qoZPsaHpn0Y/Qk1Vu26hNqXo7+nHdli+sLsXINvS1f8kR3NKhGKmA==}
- hasBin: true
- peerDependencies:
- ws: ^8.18.0
- zod: ^3.23.8
- peerDependenciesMeta:
- ws:
- optional: true
- zod:
- optional: true
-
resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}
@@ -5627,10 +5583,6 @@ packages:
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
- resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
- engines: {node: '>=6'}
-
resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}
@@ -5657,9 +5609,6 @@ packages:
resolution: {integrity: sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==}
- resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}
-
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
engines: {node: '>=8'}
@@ -5686,10 +5635,6 @@ packages:
resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}
- resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==}
- engines: {node: '>=8'}
-
resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}
@@ -6012,10 +5957,6 @@ packages:
resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==}
engines: {node: '>=8'}
- resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}
- engines: {node: '>=4'}
-
resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}
engines: {node: '>=8'}
@@ -6457,10 +6398,6 @@ packages:
resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==}
engines: {node: '>= 14.0.0'}
- resolution: {integrity: sha512-rqy30BSpxPznbbTcAcci90oZ1YR4DqvKcNXNerG5gQBU2v4jk0cygheiul5J6ExIMrgDVuanv/MkGfqZbKrNNg==}
- engines: {node: 10.* || >= 12.*}
-
resolution: {integrity: sha512-YiHwDhSvCiItoAgsKtoLFCuakDzDsJ1DLDnSouTaTmdOcOwIkSzbLXduaQ6M5DRVhuZC/NYaaZ/mtHbWMv/S6Q==}
engines: {node: ^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0}
@@ -6485,9 +6422,6 @@ packages:
esbuild:
optional: true
- resolution: {integrity: sha512-pSOfUMx8Ld/WUreoSzvMFQG5i9uEiWIsBYjpU9+TTASOeUa89j5HykomeqVULm1oqWtBdleI3KEFRLrlA3zGIw==}
-
resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}
hasBin: true
@@ -6502,10 +6436,6 @@ packages:
'@swc/wasm':
optional: true
- resolution: {integrity: sha512-iS47YTbdIcvN8Nh/1BFyziyUqmjXz7GVzWu02RaZXqb+e/3Qe1B7IQ4860krOeCGUeJmterAlaM2FRH0Ue0hjw==}
- hasBin: true
-
resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}
engines: {node: '>=12.12.0'}
@@ -6945,10 +6875,6 @@ packages:
resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==}
- resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
- engines: {node: '>= 6'}
-
resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==}
engines: {node: '>= 14.6'}
@@ -10651,13 +10577,6 @@ snapshots:
'@tootallnate/[email protected]': {}
- '@ts-morph/[email protected]':
- dependencies:
- fast-glob: 3.3.3
- minimatch: 3.1.2
- mkdirp: 1.0.4
- path-browserify: 1.0.1
-
'@tsconfig/[email protected]': {}
'@tsconfig/[email protected]': {}
@@ -10796,8 +10715,6 @@ snapshots:
dependencies:
'@types/node': 20.14.1
- '@types/[email protected]': {}
-
'@types/[email protected]': {}
@@ -11923,8 +11840,6 @@ snapshots:
-
@@ -11968,8 +11883,6 @@ snapshots:
-
@@ -12002,14 +11915,6 @@ snapshots:
object-assign: 4.1.1
vary: 1.1.2
- dependencies:
- '@types/parse-json': 4.0.2
- import-fresh: 3.3.1
- parse-json: 5.2.0
- path-type: 4.0.0
- yaml: 1.10.2
-
@@ -12864,11 +12769,6 @@ snapshots:
- dependencies:
- parent-module: 1.0.1
- resolve-from: 4.0.0
-
dependencies:
acorn: 8.15.0
@@ -13770,8 +13670,6 @@ snapshots:
optionalDependencies:
typescript: 5.8.3
-
@@ -13925,11 +13823,6 @@ snapshots:
dependencies:
mimic-function: 5.0.1
- optionalDependencies:
- zod: 3.24.2
-
@@ -14066,10 +13959,6 @@ snapshots:
- dependencies:
- callsites: 3.1.0
-
@@ -14101,8 +13990,6 @@ snapshots:
-
@@ -14120,8 +14007,6 @@ snapshots:
-
dependencies:
through: 2.3.8
@@ -14447,8 +14332,6 @@ snapshots:
dependencies:
resolve-from: 5.0.0
-
@@ -14912,8 +14795,6 @@ snapshots:
-
dependencies:
bs-logger: 0.2.6
@@ -14932,11 +14813,6 @@ snapshots:
'@jest/types': 29.6.3
babel-jest: 29.7.0(@babel/[email protected])
- dependencies:
- '@ts-morph/common': 0.12.3
- code-block-writer: 11.0.3
-
dependencies:
'@cspotcode/source-map-support': 0.8.1
@@ -14955,15 +14831,6 @@ snapshots:
v8-compile-cache-lib: 3.0.1
yn: 3.1.1
- dependencies:
- commander: 6.2.1
- cosmiconfig: 7.1.0
- json5: 2.2.3
- lodash: 4.17.21
- true-myth: 4.1.1
- ts-morph: 13.0.3
-
dependencies:
cross-spawn: 7.0.6
@@ -15458,8 +15325,6 @@ snapshots:
-
diff --git a/apps/api/src/lib/extract/archive/crawling-index.ts b/apps/api/src/lib/extract/archive/crawling-index.ts
deleted file mode 100644
index 19fa94957..000000000
--- a/apps/api/src/lib/extract/archive/crawling-index.ts
+++ /dev/null
@@ -1,81 +0,0 @@
-// const id = crypto.randomUUID();
-
-// const sc: StoredCrawl = {
-// originUrl: request.urls[0].replace("/*",""),
-// crawlerOptions: toLegacyCrawlerOptions({
-// maxDepth: 15,
-// limit: 5000,
-// includePaths: [],
-// excludePaths: [],
-// ignoreSitemap: false,
-// allowExternalLinks: false,
-// allowBackwardLinks: true,
-// allowSubdomains: false,
-// ignoreRobotsTxt: false,
-// deduplicateSimilarURLs: false,
-// ignoreQueryParameters: false
-// }),
-// scrapeOptions: {
-// formats: ["markdown"],
-// onlyMainContent: true,
-// waitFor: 0,
-// mobile: false,
-// removeBase64Images: true,
-// fastMode: false,
-// parsePDF: true,
-// skipTlsVerification: false,
-// },
-// internalOptions: {
-// disableSmartWaitCache: true,
-// isBackgroundIndex: true
-// },
-// team_id: process.env.BACKGROUND_INDEX_TEAM_ID!,
-// createdAt: Date.now(),
-// plan: "hobby", // make it a low concurrency
-// };
-
-// // Save the crawl configuration
-// await saveCrawl(id, sc);
-
-// // Then kick off the job
-// await _addScrapeJobToBullMQ({
-// url: request.urls[0].replace("/*",""),
-// mode: "kickoff" as const,
-// team_id: process.env.BACKGROUND_INDEX_TEAM_ID!,
-// plan: "hobby", // make it a low concurrency
-// crawlerOptions: sc.crawlerOptions,
-// scrapeOptions: sc.scrapeOptions,
-// internalOptions: sc.internalOptions,
-// origin: "index",
-// crawl_id: id,
-// webhook: null,
-// v1: true,
-// }, {}, crypto.randomUUID(), 50);
-
-// we restructure and make all of the arrays we need to fill into objects,
-// adding them to a single object so the llm can fill them one at a time
-// TODO: make this work for more complex schemas where arrays are not first level
-
-// let schemasForLLM: {} = {};
-// for (const key in largeArraysSchema) {
-// const originalSchema = structuredClone(largeArraysSchema[key].items);
-// console.log(
-// "key",
-// key,
-// "\noriginalSchema",
-// JSON.stringify(largeArraysSchema[key], null, 2),
-// );
-// let clonedObj = {
-// type: "object",
-// properties: {
-// informationFilled: {
-// type: "boolean",
-// },
-// data: {
-// type: "object",
-// properties: originalSchema.properties,
-// },
-// },
-// };
-// schemasForLLM[key] = clonedObj;
-// }
diff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.ts
deleted file mode 100644
index 25b22e510..000000000
--- a/apps/api/src/lib/extract/completions.ts
+++ /dev/null
@@ -1,124 +0,0 @@
-// use llmExtract.ts instead
-
-// import OpenAI from "openai";
-// import { encoding_for_model } from "@dqbd/tiktoken";
-// import { TiktokenModel } from "@dqbd/tiktoken";
-// import { ExtractOptions } from "../../controllers/v1/types";
-// import { Document } from "../entities";
-// import { z } from "zod";
-
-// const maxTokens = 32000;
-// const modifier = 4;
-
-// export class LLMRefusalError extends Error {
-// constructor(refusal: string) {
-// super("LLM refused to extract the website's content");
-// this.name = "LLMRefusalError";
-// }
-// }
-
-// interface GenerateCompletionsParams {
-// systemPrompt?: string;
-// prompt?: string;
-// schema?: any;
-// pagesContent: string;
-// }
-
-// export async function generateBasicCompletion(prompt: string) {
-// const openai = new OpenAI();
-// const model: TiktokenModel =
-// (process.env.MODEL_NAME as TiktokenModel) || "gpt-4o-mini";
-
-// const completion = await openai.chat.completions.create({
-// model,
-// messages: [{ role: "user", content: prompt }],
-// });
-
-// return completion.choices[0].message.content;
-// }
-
-// export async function generateFinalExtraction({
-// pagesContent,
-// systemPrompt,
-// prompt,
-// schema,
-// }: GenerateCompletionsParams): Promise<{
-// content: string;
-// metadata: { numTokens: number; warning: string };
-// }> {
-// const openai = new OpenAI();
-// const model: TiktokenModel =
-// (process.env.MODEL_NAME as TiktokenModel) || "gpt-4o-mini";
-
-// let extractionContent = pagesContent;
-// let numTokens = 0;
-// let warning = "";
-
-// const encoder = encoding_for_model(model);
-// try {
-// const tokens = encoder.encode(extractionContent);
-// numTokens = tokens.length;
-// } catch (error) {
-// extractionContent = extractionContent.slice(0, maxTokens * modifier);
-// warning = `Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (${maxTokens}) we support.`;
-// } finally {
-// encoder.free();
-// }
-
-// if (numTokens > maxTokens) {
-// extractionContent = extractionContent.slice(0, maxTokens * modifier);
-// warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;
-// }
-
-// if (schema && (schema.type === "array" || schema._type === "ZodArray")) {
-// schema = {
-// type: "object",
-// properties: {
-// items: schema,
-// },
-// required: ["items"],
-// additionalProperties: false,
-// };
-// } else if (schema) {
-// schema.additionalProperties = false;
-// schema.required = Object.keys(schema.properties);
-// }
-
-// const jsonCompletion = await openai.beta.chat.completions.parse({
-// temperature: 0,
-// model,
-// messages: [
-// { role: "system", content: systemPrompt ?? "" },
-// { role: "user", content: [{ type: "text", text: extractionContent }] },
-// {
-// role: "user",
-// content: prompt
-// ? `Transform the above content into structured JSON output based on the following user request: ${prompt}`
-// : "Transform the above content into structured JSON output.",
-// },
-// ],
-// response_format: schema
-// ? {
-// type: "json_schema",
-// json_schema: {
-// name: "websiteContent",
-// schema: schema,
-// strict: true,
-// },
-// }
-// : { type: "json_object" },
-// });
-
-// if (jsonCompletion.choices[0].message.refusal !== null && jsonCompletion.choices[0].message.refusal !== undefined) {
-// throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal);
-// }
-
-// const extraction = jsonCompletion.choices[0].message.parsed;
-// return {
-// content: extraction ?? "",
-// metadata: {
-// numTokens,
-// warning,
-// },
-// };
-// }
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts
index a33a061d4..c161666de 100644
--- a/apps/api/src/lib/extract/extract-redis.ts
+++ b/apps/api/src/lib/extract/extract-redis.ts
@@ -12,7 +12,6 @@ export enum ExtractStep {
MULTI_ENTITY_EXTRACT = "multi-entity-extract",
SCRAPE = "scrape",
EXTRACT = "extract",
- COMPLETE = "complete",
}
type ExtractedStep = {
diff --git a/apps/api/src/lib/extract/fire-0/ranker-f0.ts b/apps/api/src/lib/extract/fire-0/ranker-f0.ts
deleted file mode 100644
index 0ef2eff8f..000000000
--- a/apps/api/src/lib/extract/fire-0/ranker-f0.ts
+++ /dev/null
@@ -1,100 +0,0 @@
-import { embed } from "ai";
-import { configDotenv } from "dotenv";
-import { getEmbeddingModel } from "../../../lib/generic-ai";
-
-configDotenv();
-
-async function getEmbedding(
- text: string,
- metadata: { teamId: string; extractId?: string },
-) {
- const { embedding } = await embed({
- model: getEmbeddingModel("text-embedding-3-small"),
- value: text,
- experimental_telemetry: {
- isEnabled: true,
- metadata: {
- ...(metadata.extractId
- ? {
- langfuseTraceId: "extract:" + metadata.extractId,
- extractId: metadata.extractId,
- }
- : {}),
- teamId: metadata.teamId,
- },
- },
- });
-
- return embedding;
-}
-
-const cosineSimilarity = (vec1: number[], vec2: number[]): number => {
- const dotProduct = vec1.reduce((sum, val, i) => sum + val * vec2[i], 0);
- const magnitude1 = Math.sqrt(vec1.reduce((sum, val) => sum + val * val, 0));
- const magnitude2 = Math.sqrt(vec2.reduce((sum, val) => sum + val * val, 0));
- if (magnitude1 === 0 || magnitude2 === 0) return 0;
- return dotProduct / (magnitude1 * magnitude2);
-};
-
-// Function to convert text to vector
-const textToVector = (searchQuery: string, text: string): number[] => {
- const words = searchQuery.toLowerCase().split(/\W+/);
- return words.map(word => {
- const count = (text.toLowerCase().match(new RegExp(word, "g")) || [])
- .length;
- return count / text.length;
- });
-};
-
-async function performRanking_F0(
- linksWithContext: string[],
- links: string[],
- searchQuery: string,
- metadata: { teamId: string; extractId?: string },
-) {
- try {
- // Handle invalid inputs
- if (!searchQuery || !linksWithContext.length || !links.length) {
- return [];
- }
-
- // Sanitize search query by removing null characters
- const sanitizedQuery = searchQuery;
-
- // Generate embeddings for the search query
- const queryEmbedding = await getEmbedding(sanitizedQuery, metadata);
-
- // Generate embeddings for each link and calculate similarity in parallel
- const linksAndScores = await Promise.all(
- linksWithContext.map((linkWithContext, index) =>
- getEmbedding(linkWithContext, metadata)
- .then(linkEmbedding => {
- const score = cosineSimilarity(queryEmbedding, linkEmbedding);
- return {
- link: links[index],
- linkWithContext,
- score,
- originalIndex: index,
- };
- })
- .catch(() => ({
- link: links[index],
- linkWithContext,
- score: 0,
- originalIndex: index,
- })),
- ),
- );
-
- // Sort links based on similarity scores while preserving original order for equal scores
- linksAndScores.sort((a, b) => {
- const scoreDiff = b.score - a.score;
- return scoreDiff === 0 ? a.originalIndex - b.originalIndex : scoreDiff;
- });
-
- return linksAndScores;
- } catch (error) {
- console.error(`Error performing semantic search: ${error}`);
- return [];
- }
-}
diff --git a/apps/api/src/run-req.ts b/apps/api/src/run-req.ts
deleted file mode 100644
index 60751c77d..000000000
--- a/apps/api/src/run-req.ts
+++ /dev/null
@@ -1,175 +0,0 @@
-import axios from "axios";
-import { promises as fs } from "fs";
-import { v4 as uuidV4 } from "uuid";
-
-interface Result {
- start_url: string;
- job_id?: string;
- idempotency_key?: string;
- result_data_jsonb?: any;
-}
-
-async function sendCrawl(result: Result): Promise<string | undefined> {
- const idempotencyKey = uuidV4();
- const url = result.start_url;
- try {
- const response = await axios.post(
- "https://staging-firecrawl-scraper-js.fly.dev/v0/crawl",
- {
- url: url,
- crawlerOptions: {
- limit: 75,
- },
- pageOptions: {
- includeHtml: true,
- replaceAllPathsWithAbsolutePaths: true,
- waitFor: 1000,
- },
- },
- {
- headers: {
- "Content-Type": "application/json",
- Authorization: `Bearer `,
- },
- },
- );
- result.idempotency_key = idempotencyKey;
- return response.data.jobId;
- } catch (error) {
- console.error("Error sending crawl:", error);
- return undefined;
- }
-}
-
-async function getContent(result: Result): Promise<boolean> {
- let attempts = 0;
- while (attempts < 120) {
- // Reduce the number of attempts to speed up
- try {
- const response = await axios.get(
- `https://staging-firecrawl-scraper-js.fly.dev/v0/crawl/status/${result.job_id}`,
- {
- headers: {
- "Content-Type": "application/json",
- Authorization: `Bearer `,
- },
- },
- );
- if (response.data.status === "completed") {
- result.result_data_jsonb = response.data.data;
- // Job actually completed
- return true;
- }
- } catch (error) {
- console.error("Error getting content:", error);
- }
- const randomSleep = Math.floor(Math.random() * 15000) + 5000;
- await new Promise(resolve => setTimeout(resolve, randomSleep)); // Reduce sleep time to 1.5 seconds
- attempts++;
- }
- // Set result as null if timed out
- result.result_data_jsonb = null;
- return false;
-}
-
-async function processResults(results: Result[]): Promise<void> {
- let processedCount = 0;
- let starterCount = 0;
- const queue: Result[] = [];
- const processedUrls = new Set<string>();
-
- // Initialize the queue with the first 1000 results
- for (let i = 0; i < Math.min(100, results.length); i++) {
- queue.push(results[i]);
- processedUrls.add(results[i].start_url);
- }
-
- // Function to process a single result
- const processSingleResult = async (result: Result) => {
- const jobId = await sendCrawl(result);
- if (jobId) {
- console.log(`Job requested count: ${starterCount}`);
- starterCount++;
- result.job_id = jobId;
- processedCount++;
- // Save the result to the file
- try {
- // Save job id along with the start_url
- const resultWithJobId = results.map(r => ({
- start_url: r.start_url,
- job_id: r.job_id,
- }));
- await fs.writeFile(
- "results_with_job_id_4000_6000.json",
- JSON.stringify(resultWithJobId, null, 4),
- );
- } catch (error) {
- console.error("Error writing to results_with_content.json:", error);
- }
-
- // Add a new result to the queue if there are more results to process
- // if (processedCount < results.length) {
- // for (let i = queue.length; i < results.length; i++) {
- // if (!processedUrls.has(results[i].start_url)) {
- // const nextResult = results[i];
- // console.log("Next result:", nextResult.start_url);
- // queue.push(nextResult);
- // processedUrls.add(nextResult.start_url);
- // console.log(`Queue length: ${queue.length}`);
- // processSingleResult(nextResult);
- // break;
- // }
- // }
- // }
- }
- };
-
- // Start processing the initial queue concurrently
- // for (let i = 0; i < queue.length; i++) {
- // processSingleResult(queue[i]);
- // if ((i + 1) % 500 === 0) {
- // console.log(`Processed ${i + 1} results, waiting for 1 minute before adding the next batch...`);
- // await new Promise(resolve => setTimeout(resolve, 60 * 1000)); // Wait for 1 minute
- // }
- // }
- // Start processing the initial queue concurrently
- // await Promise.all(queue.map(result => processSingleResult(result)));
- for (let i = 0; i < results.length; i += 100) {
- const batch = results.slice(i, i + 100);
- Promise.all(batch.map(result => processSingleResult(result)))
- .then(() => {
- console.log(`Processed ${i + 100} results.`);
- })
- .catch(error => {
- console.error(`Error processing batch starting at index ${i}:`, error);
- });
- await new Promise(resolve => setTimeout(resolve, 60 * 1000)); // Wait for 1 minute
- }
-}
-
-// Example call
-
-async function getStartUrls(): Promise<Result[]> {
- try {
- const data = await fs.readFile("starturls.json", "utf-8");
- return JSON.parse(data);
- } catch (error) {
- console.error("Error reading starturls.json:", error);
- return [];
- }
-}
-
-async function main() {
- const results: Result[] = (await getStartUrls()).slice(3999, 6000);
- // console.log(results.map((r) => r.start_url).slice(0, 3));
-
- processResults(results)
- .then(() => {
- console.log("All results processed.");
- })
- .catch(error => {
- console.error("Error processing results:", error);
- });
-}
-
-main();