Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: repalce ts-prune with knip (#2148)
tomd7541885a85e
diff --git a/apps/api/.husky/pre-commit b/apps/api/.husky/pre-commitindex 5f8a22160..629a923ce 100644--- a/apps/api/.husky/pre-commit+++ b/apps/api/.husky/pre-commit@@ -1 +1 @@-cd apps/api && pnpm lint-staged\ No newline at end of file+cd apps/api && pnpm knip --cache && pnpm lint-staged\ No newline at end of filediff --git a/apps/api/knip.config.ts b/apps/api/knip.config.tsnew file mode 100644index 000000000..09a07399e--- /dev/null+++ b/apps/api/knip.config.ts@@ -0,0 +1,13 @@+import type { KnipConfig } from "knip";++const config: KnipConfig = {+ workspaces: {+ ".": {+ entry: ["src/services/worker/**/*.ts", "src/services/**/*-worker.ts"],+ project: ["src/**/*.ts"],+ },+ },+ ignore: ["native/**"],+};++export default config;diff --git a/apps/api/package.json b/apps/api/package.jsonindex ab914b2b4..fc294a1d7 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -2,7 +2,7 @@"name": "firecrawl-scraper-js","version": "1.0.0","description": "",- "main": "index.js",+ "main": "src/index.ts","scripts": {"start": "tsc && node dist/src/harness.js --start-built","dev": "tsx src/harness.ts --start",@@ -30,7 +30,8 @@"mongo-docker-console": "docker exec -it mongodb mongosh","run-example": "npx ts-node src/example.ts","sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist",- "prepare": "cd ../.. && husky ./apps/api/.husky"+ "prepare": "cd ../.. && husky ./apps/api/.husky",+ "knip": "knip"},"author": "","license": "ISC",@@ -55,7 +56,6 @@"supertest": "^6.3.3","ts-jest": "^29.1.1","ts-node": "^10.9.1",- "ts-prune": "^0.10.3","tsc-watch": "^7.1.1","tsx": "^4.20.3","typescript": "^5.8.3"@@ -115,7 +115,6 @@"mammoth": "^1.7.2","marked": "^14.1.2","ollama-ai-provider": "^1.2.0",- "openai": "^5.20.0","parse-diff": "^0.11.1","pdf-parse": "^1.1.1","pg": "^8.16.3",@@ -161,8 +160,7 @@}},"lint-staged": {- "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write",- "*.{js,tsx,ts,tsx}": "ts-prune --skip \"native/\" --error"+ "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write"},"packageManager": "[email protected]"}diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex ea638a8ab..679108edd 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -174,9 +174,6 @@ importers:ollama-ai-provider:specifier: ^1.2.0version: 1.2.0([email protected])- openai:- specifier: ^5.20.0parse-diff:specifier: ^0.11.1version: 0.11.1@@ -304,9 +301,6 @@ importers:ts-node:specifier: ^10.9.1version: 10.9.2(@types/[email protected])([email protected])- ts-prune:- specifier: ^0.10.3- version: 0.10.3tsc-watch:specifier: ^7.1.1version: 7.1.1([email protected])@@ -3103,9 +3097,6 @@ packages:resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}engines: {node: '>= 10'}- '@ts-morph/[email protected]':- resolution: {integrity: sha512-4tUmeLyXJnJWvTFOKtcNJ1yh0a3SsTLi2MUoyj8iUNznFRN1ZquaNe7Oukqrnki2FzZkm0J9adCNLDZxUzvj+w==}-'@tsconfig/[email protected]':resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}@@ -3223,9 +3214,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-kK1eBS/Adeyis+3OlBDMeQQuasIDLUYXsi2T15ccNJ0iyUpQ4xDF7svFu3+bGVrI0CMBUclPciz+lsQR3JX3TQ==}- '@types/[email protected]':- resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==}-'@types/[email protected]':resolution: {integrity: sha512-+gbBHbNCVGGYw1S9lAIIvrHW47UYOhMIFUsJcMkMrzy1Jf0vulBN3XQIjPgnoOXveMuHnF3b57fXROnY/Or7eg==}@@ -3820,9 +3808,6 @@ packages:resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==}engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'}- resolution: {integrity: sha512-NiujjUFB4SwScJq2bwbYUtXbZhBSlY6vYzm++3Q6oC+U+injTqfPYFK8wS9COOmb2lueqp0ZRB4nK1VYeHgNyw==}-resolution: {integrity: sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==}@@ -3870,10 +3855,6 @@ packages:resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==}- resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==}- engines: {node: '>= 6'}-resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}@@ -3914,10 +3895,6 @@ packages:resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==}engines: {node: '>= 0.10'}- resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==}- engines: {node: '>=10'}-resolution: {integrity: sha512-nPklKJ5qtmY5MdBKw1NiBAoyx5Sa7p2yPpljZyQ7gyCN1m+eMFs9I6CT37Mxt8zvR5L3VzD3DJBE4WQzX3WF4A==}@@ -4684,10 +4661,6 @@ packages:resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==}- resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==}- engines: {node: '>=6'}-resolution: {integrity: sha512-5tCuY9BV8ujfOpwtAGgsTx9CGUapcFMEEyByLv1B+v2+6DhAcw+Zr0nhQT7uwaZ7DiourxFEscghOR8e1aPLQw==}@@ -5366,11 +5339,6 @@ packages:typescript:optional: true- resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}- engines: {node: '>=10'}- hasBin: true-resolution: {integrity: sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==}@@ -5545,18 +5513,6 @@ packages:resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==}engines: {node: '>=18'}- resolution: {integrity: sha512-Bmc2zLM/YWgFrDpXr9hwXqGGDdMmMpE9+qoZPsaHpn0Y/Qk1Vu26hNqXo7+nHdli+sLsXINvS1f8kR3NKhGKmA==}- hasBin: true- peerDependencies:- ws: ^8.18.0- zod: ^3.23.8- peerDependenciesMeta:- ws:- optional: true- zod:- optional: true-resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}@@ -5627,10 +5583,6 @@ packages:resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}- resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}- engines: {node: '>=6'}-resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}@@ -5657,9 +5609,6 @@ packages:resolution: {integrity: sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==}- resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}-resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}engines: {node: '>=8'}@@ -5686,10 +5635,6 @@ packages:resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==}- resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==}- engines: {node: '>=8'}-resolution: {integrity: sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==}@@ -6012,10 +5957,6 @@ packages:resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==}engines: {node: '>=8'}- resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}- engines: {node: '>=4'}-resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}engines: {node: '>=8'}@@ -6457,10 +6398,6 @@ packages:resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==}engines: {node: '>= 14.0.0'}- resolution: {integrity: sha512-rqy30BSpxPznbbTcAcci90oZ1YR4DqvKcNXNerG5gQBU2v4jk0cygheiul5J6ExIMrgDVuanv/MkGfqZbKrNNg==}- engines: {node: 10.* || >= 12.*}-resolution: {integrity: sha512-YiHwDhSvCiItoAgsKtoLFCuakDzDsJ1DLDnSouTaTmdOcOwIkSzbLXduaQ6M5DRVhuZC/NYaaZ/mtHbWMv/S6Q==}engines: {node: ^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0}@@ -6485,9 +6422,6 @@ packages:esbuild:optional: true- resolution: {integrity: sha512-pSOfUMx8Ld/WUreoSzvMFQG5i9uEiWIsBYjpU9+TTASOeUa89j5HykomeqVULm1oqWtBdleI3KEFRLrlA3zGIw==}-resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}hasBin: true@@ -6502,10 +6436,6 @@ packages:'@swc/wasm':optional: true- resolution: {integrity: sha512-iS47YTbdIcvN8Nh/1BFyziyUqmjXz7GVzWu02RaZXqb+e/3Qe1B7IQ4860krOeCGUeJmterAlaM2FRH0Ue0hjw==}- hasBin: true-resolution: {integrity: sha512-r6t37Dkk4vK44HwxOe+OzjpE/gDamZAwqXhtcAJD/hPVblcjJK45NxbK0HcDASXG0U4pEnCh640JZbeDVSC6yA==}engines: {node: '>=12.12.0'}@@ -6945,10 +6875,6 @@ packages:resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==}- resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}- engines: {node: '>= 6'}-resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==}engines: {node: '>= 14.6'}@@ -10651,13 +10577,6 @@ snapshots:'@tootallnate/[email protected]': {}- '@ts-morph/[email protected]':- dependencies:- fast-glob: 3.3.3- minimatch: 3.1.2- mkdirp: 1.0.4- path-browserify: 1.0.1-'@tsconfig/[email protected]': {}'@tsconfig/[email protected]': {}@@ -10796,8 +10715,6 @@ snapshots:dependencies:'@types/node': 20.14.1- '@types/[email protected]': {}-'@types/[email protected]': {}'@types/[email protected]':@@ -11923,8 +11840,6 @@ snapshots:- [email protected]: {}-@@ -11968,8 +11883,6 @@ snapshots:- [email protected]: {}-@@ -12002,14 +11915,6 @@ snapshots:object-assign: 4.1.1vary: 1.1.2- dependencies:- '@types/parse-json': 4.0.2- import-fresh: 3.3.1- parse-json: 5.2.0- path-type: 4.0.0- yaml: 1.10.2-@@ -12864,11 +12769,6 @@ snapshots:- dependencies:- parent-module: 1.0.1- resolve-from: 4.0.0-dependencies:acorn: 8.15.0@@ -13770,8 +13670,6 @@ snapshots:optionalDependencies:typescript: 5.8.3- [email protected]: {}-@@ -13925,11 +13823,6 @@ snapshots:dependencies:mimic-function: 5.0.1- optionalDependencies:- ws: 8.18.0([email protected])([email protected])- zod: 3.24.2-@@ -14066,10 +13959,6 @@ snapshots:- dependencies:- callsites: 3.1.0-@@ -14101,8 +13990,6 @@ snapshots:- [email protected]: {}-@@ -14120,8 +14007,6 @@ snapshots:- [email protected]: {}-dependencies:through: 2.3.8@@ -14447,8 +14332,6 @@ snapshots:dependencies:resolve-from: 5.0.0- [email protected]: {}-@@ -14912,8 +14795,6 @@ snapshots:- [email protected]: {}-[email protected](@babel/[email protected])(@jest/[email protected])(@jest/[email protected])([email protected](@babel/[email protected]))([email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])))([email protected]):dependencies:bs-logger: 0.2.6@@ -14932,11 +14813,6 @@ snapshots:'@jest/types': 29.6.3babel-jest: 29.7.0(@babel/[email protected])- dependencies:- '@ts-morph/common': 0.12.3- code-block-writer: 11.0.3-dependencies:'@cspotcode/source-map-support': 0.8.1@@ -14955,15 +14831,6 @@ snapshots:v8-compile-cache-lib: 3.0.1yn: 3.1.1- dependencies:- commander: 6.2.1- cosmiconfig: 7.1.0- json5: 2.2.3- lodash: 4.17.21- true-myth: 4.1.1- ts-morph: 13.0.3-dependencies:cross-spawn: 7.0.6@@ -15458,8 +15325,6 @@ snapshots:- [email protected]: {}-diff --git a/apps/api/src/lib/extract/archive/crawling-index.ts b/apps/api/src/lib/extract/archive/crawling-index.tsdeleted file mode 100644index 19fa94957..000000000--- a/apps/api/src/lib/extract/archive/crawling-index.ts+++ /dev/null@@ -1,81 +0,0 @@-// const id = crypto.randomUUID();--// const sc: StoredCrawl = {-// originUrl: request.urls[0].replace("/*",""),-// crawlerOptions: toLegacyCrawlerOptions({-// maxDepth: 15,-// limit: 5000,-// includePaths: [],-// excludePaths: [],-// ignoreSitemap: false,-// allowExternalLinks: false,-// allowBackwardLinks: true,-// allowSubdomains: false,-// ignoreRobotsTxt: false,-// deduplicateSimilarURLs: false,-// ignoreQueryParameters: false-// }),-// scrapeOptions: {-// formats: ["markdown"],-// onlyMainContent: true,-// waitFor: 0,-// mobile: false,-// removeBase64Images: true,-// fastMode: false,-// parsePDF: true,-// skipTlsVerification: false,-// },-// internalOptions: {-// disableSmartWaitCache: true,-// isBackgroundIndex: true-// },-// team_id: process.env.BACKGROUND_INDEX_TEAM_ID!,-// createdAt: Date.now(),-// plan: "hobby", // make it a low concurrency-// };--// // Save the crawl configuration-// await saveCrawl(id, sc);--// // Then kick off the job-// await _addScrapeJobToBullMQ({-// url: request.urls[0].replace("/*",""),-// mode: "kickoff" as const,-// team_id: process.env.BACKGROUND_INDEX_TEAM_ID!,-// plan: "hobby", // make it a low concurrency-// crawlerOptions: sc.crawlerOptions,-// scrapeOptions: sc.scrapeOptions,-// internalOptions: sc.internalOptions,-// origin: "index",-// crawl_id: id,-// webhook: null,-// v1: true,-// }, {}, crypto.randomUUID(), 50);--// we restructure and make all of the arrays we need to fill into objects,-// adding them to a single object so the llm can fill them one at a time-// TODO: make this work for more complex schemas where arrays are not first level--// let schemasForLLM: {} = {};-// for (const key in largeArraysSchema) {-// const originalSchema = structuredClone(largeArraysSchema[key].items);-// console.log(-// "key",-// key,-// "\noriginalSchema",-// JSON.stringify(largeArraysSchema[key], null, 2),-// );-// let clonedObj = {-// type: "object",-// properties: {-// informationFilled: {-// type: "boolean",-// },-// data: {-// type: "object",-// properties: originalSchema.properties,-// },-// },-// };-// schemasForLLM[key] = clonedObj;-// }diff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.tsdeleted file mode 100644index 25b22e510..000000000--- a/apps/api/src/lib/extract/completions.ts+++ /dev/null@@ -1,124 +0,0 @@-// use llmExtract.ts instead--// import OpenAI from "openai";-// import { encoding_for_model } from "@dqbd/tiktoken";-// import { TiktokenModel } from "@dqbd/tiktoken";-// import { ExtractOptions } from "../../controllers/v1/types";-// import { Document } from "../entities";-// import { z } from "zod";--// const maxTokens = 32000;-// const modifier = 4;--// export class LLMRefusalError extends Error {-// constructor(refusal: string) {-// super("LLM refused to extract the website's content");-// this.name = "LLMRefusalError";-// }-// }--// interface GenerateCompletionsParams {-// systemPrompt?: string;-// prompt?: string;-// schema?: any;-// pagesContent: string;-// }--// export async function generateBasicCompletion(prompt: string) {-// const openai = new OpenAI();-// const model: TiktokenModel =-// (process.env.MODEL_NAME as TiktokenModel) || "gpt-4o-mini";--// const completion = await openai.chat.completions.create({-// model,-// messages: [{ role: "user", content: prompt }],-// });--// return completion.choices[0].message.content;-// }--// export async function generateFinalExtraction({-// pagesContent,-// systemPrompt,-// prompt,-// schema,-// }: GenerateCompletionsParams): Promise<{-// content: string;-// metadata: { numTokens: number; warning: string };-// }> {-// const openai = new OpenAI();-// const model: TiktokenModel =-// (process.env.MODEL_NAME as TiktokenModel) || "gpt-4o-mini";--// let extractionContent = pagesContent;-// let numTokens = 0;-// let warning = "";--// const encoder = encoding_for_model(model);-// try {-// const tokens = encoder.encode(extractionContent);-// numTokens = tokens.length;-// } catch (error) {-// extractionContent = extractionContent.slice(0, maxTokens * modifier);-// warning = `Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (${maxTokens}) we support.`;-// } finally {-// encoder.free();-// }--// if (numTokens > maxTokens) {-// extractionContent = extractionContent.slice(0, maxTokens * modifier);-// warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;-// }--// if (schema && (schema.type === "array" || schema._type === "ZodArray")) {-// schema = {-// type: "object",-// properties: {-// items: schema,-// },-// required: ["items"],-// additionalProperties: false,-// };-// } else if (schema) {-// schema.additionalProperties = false;-// schema.required = Object.keys(schema.properties);-// }--// const jsonCompletion = await openai.beta.chat.completions.parse({-// temperature: 0,-// model,-// messages: [-// { role: "system", content: systemPrompt ?? "" },-// { role: "user", content: [{ type: "text", text: extractionContent }] },-// {-// role: "user",-// content: prompt-// ? `Transform the above content into structured JSON output based on the following user request: ${prompt}`-// : "Transform the above content into structured JSON output.",-// },-// ],-// response_format: schema-// ? {-// type: "json_schema",-// json_schema: {-// name: "websiteContent",-// schema: schema,-// strict: true,-// },-// }-// : { type: "json_object" },-// });--// if (jsonCompletion.choices[0].message.refusal !== null && jsonCompletion.choices[0].message.refusal !== undefined) {-// throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal);-// }--// const extraction = jsonCompletion.choices[0].message.parsed;-// return {-// content: extraction ?? "",-// metadata: {-// numTokens,-// warning,-// },-// };-// }diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.tsindex a33a061d4..c161666de 100644--- a/apps/api/src/lib/extract/extract-redis.ts+++ b/apps/api/src/lib/extract/extract-redis.ts@@ -12,7 +12,6 @@ export enum ExtractStep {MULTI_ENTITY_EXTRACT = "multi-entity-extract",SCRAPE = "scrape",EXTRACT = "extract",- COMPLETE = "complete",}type ExtractedStep = {diff --git a/apps/api/src/lib/extract/fire-0/ranker-f0.ts b/apps/api/src/lib/extract/fire-0/ranker-f0.tsdeleted file mode 100644index 0ef2eff8f..000000000--- a/apps/api/src/lib/extract/fire-0/ranker-f0.ts+++ /dev/null@@ -1,100 +0,0 @@-import { embed } from "ai";-import { configDotenv } from "dotenv";-import { getEmbeddingModel } from "../../../lib/generic-ai";--configDotenv();--async function getEmbedding(- text: string,- metadata: { teamId: string; extractId?: string },-) {- const { embedding } = await embed({- model: getEmbeddingModel("text-embedding-3-small"),- value: text,- experimental_telemetry: {- isEnabled: true,- metadata: {- ...(metadata.extractId- ? {- langfuseTraceId: "extract:" + metadata.extractId,- extractId: metadata.extractId,- }- : {}),- teamId: metadata.teamId,- },- },- });-- return embedding;-}--const cosineSimilarity = (vec1: number[], vec2: number[]): number => {- const dotProduct = vec1.reduce((sum, val, i) => sum + val * vec2[i], 0);- const magnitude1 = Math.sqrt(vec1.reduce((sum, val) => sum + val * val, 0));- const magnitude2 = Math.sqrt(vec2.reduce((sum, val) => sum + val * val, 0));- if (magnitude1 === 0 || magnitude2 === 0) return 0;- return dotProduct / (magnitude1 * magnitude2);-};--// Function to convert text to vector-const textToVector = (searchQuery: string, text: string): number[] => {- const words = searchQuery.toLowerCase().split(/\W+/);- return words.map(word => {- const count = (text.toLowerCase().match(new RegExp(word, "g")) || [])- .length;- return count / text.length;- });-};--async function performRanking_F0(- linksWithContext: string[],- links: string[],- searchQuery: string,- metadata: { teamId: string; extractId?: string },-) {- try {- // Handle invalid inputs- if (!searchQuery || !linksWithContext.length || !links.length) {- return [];- }-- // Sanitize search query by removing null characters- const sanitizedQuery = searchQuery;-- // Generate embeddings for the search query- const queryEmbedding = await getEmbedding(sanitizedQuery, metadata);-- // Generate embeddings for each link and calculate similarity in parallel- const linksAndScores = await Promise.all(- linksWithContext.map((linkWithContext, index) =>- getEmbedding(linkWithContext, metadata)- .then(linkEmbedding => {- const score = cosineSimilarity(queryEmbedding, linkEmbedding);- return {- link: links[index],- linkWithContext,- score,- originalIndex: index,- };- })- .catch(() => ({- link: links[index],- linkWithContext,- score: 0,- originalIndex: index,- })),- ),- );-- // Sort links based on similarity scores while preserving original order for equal scores- linksAndScores.sort((a, b) => {- const scoreDiff = b.score - a.score;- return scoreDiff === 0 ? a.originalIndex - b.originalIndex : scoreDiff;- });-- return linksAndScores;- } catch (error) {- console.error(`Error performing semantic search: ${error}`);- return [];- }-}diff --git a/apps/api/src/run-req.ts b/apps/api/src/run-req.tsdeleted file mode 100644index 60751c77d..000000000--- a/apps/api/src/run-req.ts+++ /dev/null@@ -1,175 +0,0 @@-import axios from "axios";-import { promises as fs } from "fs";-import { v4 as uuidV4 } from "uuid";--interface Result {- start_url: string;- job_id?: string;- idempotency_key?: string;- result_data_jsonb?: any;-}--async function sendCrawl(result: Result): Promise<string | undefined> {- const idempotencyKey = uuidV4();- const url = result.start_url;- try {- const response = await axios.post(- "https://staging-firecrawl-scraper-js.fly.dev/v0/crawl",- {- url: url,- crawlerOptions: {- limit: 75,- },- pageOptions: {- includeHtml: true,- replaceAllPathsWithAbsolutePaths: true,- waitFor: 1000,- },- },- {- headers: {- "Content-Type": "application/json",- Authorization: `Bearer `,- },- },- );- result.idempotency_key = idempotencyKey;- return response.data.jobId;- } catch (error) {- console.error("Error sending crawl:", error);- return undefined;- }-}--async function getContent(result: Result): Promise<boolean> {- let attempts = 0;- while (attempts < 120) {- // Reduce the number of attempts to speed up- try {- const response = await axios.get(- `https://staging-firecrawl-scraper-js.fly.dev/v0/crawl/status/${result.job_id}`,- {- headers: {- "Content-Type": "application/json",- Authorization: `Bearer `,- },- },- );- if (response.data.status === "completed") {- result.result_data_jsonb = response.data.data;- // Job actually completed- return true;- }- } catch (error) {- console.error("Error getting content:", error);- }- const randomSleep = Math.floor(Math.random() * 15000) + 5000;- await new Promise(resolve => setTimeout(resolve, randomSleep)); // Reduce sleep time to 1.5 seconds- attempts++;- }- // Set result as null if timed out- result.result_data_jsonb = null;- return false;-}--async function processResults(results: Result[]): Promise<void> {- let processedCount = 0;- let starterCount = 0;- const queue: Result[] = [];- const processedUrls = new Set<string>();-- // Initialize the queue with the first 1000 results- for (let i = 0; i < Math.min(100, results.length); i++) {- queue.push(results[i]);- processedUrls.add(results[i].start_url);- }-- // Function to process a single result- const processSingleResult = async (result: Result) => {- const jobId = await sendCrawl(result);- if (jobId) {- console.log(`Job requested count: ${starterCount}`);- starterCount++;- result.job_id = jobId;- processedCount++;- // Save the result to the file- try {- // Save job id along with the start_url- const resultWithJobId = results.map(r => ({- start_url: r.start_url,- job_id: r.job_id,- }));- await fs.writeFile(- "results_with_job_id_4000_6000.json",- JSON.stringify(resultWithJobId, null, 4),- );- } catch (error) {- console.error("Error writing to results_with_content.json:", error);- }-- // Add a new result to the queue if there are more results to process- // if (processedCount < results.length) {- // for (let i = queue.length; i < results.length; i++) {- // if (!processedUrls.has(results[i].start_url)) {- // const nextResult = results[i];- // console.log("Next result:", nextResult.start_url);- // queue.push(nextResult);- // processedUrls.add(nextResult.start_url);- // console.log(`Queue length: ${queue.length}`);- // processSingleResult(nextResult);- // break;- // }- // }- // }- }- };-- // Start processing the initial queue concurrently- // for (let i = 0; i < queue.length; i++) {- // processSingleResult(queue[i]);- // if ((i + 1) % 500 === 0) {- // console.log(`Processed ${i + 1} results, waiting for 1 minute before adding the next batch...`);- // await new Promise(resolve => setTimeout(resolve, 60 * 1000)); // Wait for 1 minute- // }- // }- // Start processing the initial queue concurrently- // await Promise.all(queue.map(result => processSingleResult(result)));- for (let i = 0; i < results.length; i += 100) {- const batch = results.slice(i, i + 100);- Promise.all(batch.map(result => processSingleResult(result)))- .then(() => {- console.log(`Processed ${i + 100} results.`);- })- .catch(error => {- console.error(`Error processing batch starting at index ${i}:`, error);- });- await new Promise(resolve => setTimeout(resolve, 60 * 1000)); // Wait for 1 minute- }-}--// Example call--async function getStartUrls(): Promise<Result[]> {- try {- const data = await fs.readFile("starturls.json", "utf-8");- return JSON.parse(data);- } catch (error) {- console.error("Error reading starturls.json:", error);- return [];- }-}--async function main() {- const results: Result[] = (await getStartUrls()).slice(3999, 6000);- // console.log(results.map((r) => r.start_url).slice(0, 3));-- processResults(results)- .then(() => {- console.log("All results processed.");- })- .catch(error => {- console.error("Error processing results:", error);- });-}--main();