Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): update package.json and pnpm-lock.yaml to allow specific scripts and upgrade debug dependency to version 4.4.3; refactor URL schema export in batch-scrape controller
AdemÃlson Tonato3065838555b6
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 9c8a7604e..7f069bc4a 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -149,7 +149,12 @@]},"pnpm": {+ "allowScripts": {+ "@coinbase/x402": true,+ "oxc-resolver": true+ },"onlyBuiltDependencies": [+ "@coinbase/x402","@mendable/firecrawl-rs","@sentry-internal/node-cpu-profiler","@sentry/cli",@@ -160,13 +165,15 @@"koffi","libpq","msgpackr-extract",+ "oxc-resolver","protobufjs","supabase","utf-8-validate","wordpos"],"overrides": {- "brace-expansion@>=2.0.0 <=2.0.1": "2.0.2"+ "brace-expansion@>=2.0.0 <=2.0.1": "2.0.2",+ "debug": ">=4.4.3"}},"lint-staged": {diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 00bd861c2..ece1d70e6 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -7,6 +7,7 @@ settings:overrides:brace-expansion@>=2.0.0 <=2.0.1: 2.0.2+ debug: '>=4.4.3'importers:@@ -4010,33 +4011,8 @@ packages:resolution: {integrity: sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==}- resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}- peerDependencies:- supports-color: '*'- peerDependenciesMeta:- supports-color:- optional: true-- resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==}- peerDependencies:- supports-color: '*'- peerDependenciesMeta:- supports-color:- optional: true-- resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==}- engines: {node: '>=6.0'}- peerDependencies:- supports-color: '*'- peerDependenciesMeta:- supports-color:- optional: true-- resolution: {integrity: sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==}+ resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==}engines: {node: '>=6.0'}peerDependencies:supports-color: '*'@@ -5390,12 +5366,6 @@ packages:resolution: {integrity: sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==}- resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}-- resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}-resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}@@ -7245,7 +7215,7 @@ snapshots:'@babel/traverse': 7.24.6'@babel/types': 7.24.6convert-source-map: 2.0.0- debug: 4.4.1+ debug: 4.4.3gensync: 1.0.0-beta.2json5: 2.2.3semver: 6.3.1@@ -7434,7 +7404,7 @@ snapshots:'@babel/helper-split-export-declaration': 7.24.6'@babel/parser': 7.24.6'@babel/types': 7.24.6- debug: 4.4.1+ debug: 4.4.3globals: 11.12.0transitivePeerDependencies:- supports-color@@ -8227,7 +8197,7 @@ snapshots:bufferutil: 4.0.9cross-fetch: 4.1.0([email protected])date-fns: 2.30.0- debug: 4.4.1+ debug: 4.4.3eciesjs: 0.4.15eventemitter2: 6.4.9readable-stream: 3.6.2@@ -8251,7 +8221,7 @@ snapshots:'@paulmillr/qr': 0.2.1bowser: 2.11.0cross-fetch: 4.1.0([email protected])- debug: 4.4.1+ debug: 4.4.3eciesjs: 0.4.15eth-rpc-errors: 4.0.3eventemitter2: 6.4.9@@ -8274,7 +8244,7 @@ snapshots:dependencies:'@ethereumjs/tx': 4.2.0'@types/debug': 4.1.12- debug: 4.4.1+ debug: 4.4.3semver: 7.7.2superstruct: 1.0.4transitivePeerDependencies:@@ -8287,7 +8257,7 @@ snapshots:'@noble/hashes': 1.8.0'@scure/base': 1.2.6'@types/debug': 4.1.12- debug: 4.4.1+ debug: 4.4.3pony-cause: 2.1.11semver: 7.7.2uuid: 9.0.1@@ -8301,7 +8271,7 @@ snapshots:'@noble/hashes': 1.8.0'@scure/base': 1.2.6'@types/debug': 4.1.12- debug: 4.4.1+ debug: 4.4.3pony-cause: 2.1.11semver: 7.7.2uuid: 9.0.1@@ -8336,7 +8306,7 @@ snapshots:'@octokit/rest': 22.0.0clipanion: 4.0.0-rc.4([email protected])colorette: 2.0.20- debug: 4.4.1+ debug: 4.4.3es-toolkit: 1.39.10find-up: 7.0.0js-yaml: 4.1.0@@ -8362,7 +8332,7 @@ snapshots:dependencies:'@napi-rs/lzma': 1.4.5'@napi-rs/tar': 1.1.0- debug: 4.4.1+ debug: 4.4.3transitivePeerDependencies:- supports-color@@ -11573,7 +11543,7 @@ snapshots:dependencies:- debug: 4.4.1+ debug: 4.4.3transitivePeerDependencies:- supports-color@@ -11773,7 +11743,7 @@ snapshots:dependencies:bytes: 3.1.2content-type: 1.0.5- debug: 2.6.9+ debug: 4.4.3depd: 2.0.0destroy: 1.2.0http-errors: 2.0.0@@ -12136,19 +12106,7 @@ snapshots:- dependencies:- ms: 2.0.0-- dependencies:- ms: 2.1.3-- dependencies:- ms: 2.1.2-dependencies:ms: 2.1.3@@ -12300,7 +12258,7 @@ snapshots:dependencies:'@socket.io/component-emitter': 3.1.2- debug: 4.3.5+ debug: 4.4.3engine.io-parser: 5.2.3ws: 8.17.1([email protected])([email protected])xmlhttprequest-ssl: 2.1.2@@ -12482,7 +12440,7 @@ snapshots:content-type: 1.0.5cookie: 0.7.1cookie-signature: 1.0.6- debug: 2.6.9+ debug: 4.4.3depd: 2.0.0encodeurl: 2.0.0escape-html: 1.0.3@@ -12583,7 +12541,7 @@ snapshots:dependencies:- debug: 2.6.9+ debug: 4.4.3encodeurl: 2.0.0escape-html: 1.0.3on-finished: 2.4.1@@ -12906,28 +12864,28 @@ snapshots:dependencies:'@tootallnate/once': 2.0.0agent-base: 6.0.2- debug: 4.4.1+ debug: 4.4.3transitivePeerDependencies:- supports-colordependencies:agent-base: 7.1.3- debug: 4.3.5+ debug: 4.4.3transitivePeerDependencies:- supports-colordependencies:agent-base: 6.0.2- debug: 4.3.5+ debug: 4.4.3transitivePeerDependencies:- supports-colordependencies:agent-base: 7.1.3- debug: 4.4.1+ debug: 4.4.3transitivePeerDependencies:- supports-color@@ -12982,7 +12940,7 @@ snapshots:dependencies:'@ioredis/commands': 1.2.0cluster-key-slot: 1.1.2- debug: 4.4.1+ debug: 4.4.3denque: 2.1.0lodash.defaults: 4.2.0lodash.isarguments: 3.1.0@@ -13121,7 +13079,7 @@ snapshots:dependencies:- debug: 4.4.1+ debug: 4.4.3istanbul-lib-coverage: 3.2.2source-map: 0.6.1transitivePeerDependencies:@@ -13621,7 +13579,7 @@ snapshots:dependencies:chalk: 5.6.0commander: 14.0.0- debug: 4.4.1+ debug: 4.4.3lilconfig: 3.1.3listr2: 9.0.3micromatch: 4.0.8@@ -13812,10 +13770,6 @@ snapshots:- [email protected]: {}-- [email protected]: {}-@@ -14162,7 +14116,7 @@ snapshots:dependencies:- debug: 3.2.7+ debug: 4.4.3node-ensure: 0.0.0transitivePeerDependencies:- supports-color@@ -14463,7 +14417,7 @@ snapshots:dependencies:- debug: 4.4.1+ debug: 4.4.3module-details-from-path: 1.0.3resolve: 1.22.8transitivePeerDependencies:@@ -14586,7 +14540,7 @@ snapshots:dependencies:- debug: 2.6.9+ debug: 4.4.3depd: 2.0.0destroy: 1.2.0encodeurl: 1.0.2@@ -14710,7 +14664,7 @@ snapshots:dependencies:'@socket.io/component-emitter': 3.1.2- debug: 4.3.5+ debug: 4.4.3engine.io-client: 6.6.3([email protected])([email protected])socket.io-parser: 4.2.4transitivePeerDependencies:@@ -14721,7 +14675,7 @@ snapshots:dependencies:'@socket.io/component-emitter': 3.1.2- debug: 4.3.5+ debug: 4.4.3transitivePeerDependencies:- supports-color@@ -14838,7 +14792,7 @@ snapshots:dependencies:component-emitter: 1.3.1cookiejar: 2.1.4- debug: 4.3.5+ debug: 4.4.3fast-safe-stringify: 2.1.1form-data: 4.0.4formidable: 2.1.5diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 09a6af15c..329d602db 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -4,7 +4,7 @@ import {BatchScrapeRequest,batchScrapeRequestSchema,batchScrapeRequestSchemaNoURLValidation,- url as urlSchema,+ URL as urlSchema,RequestWithAuth,ScrapeOptions,BatchScrapeResponse,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 28a681195..3d07613fd 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -63,7 +63,7 @@ const BASE_URL_SCHEMA = z.preprocess();// Standard URL schema-const URL = BASE_URL_SCHEMA;+export const URL = BASE_URL_SCHEMA;// Crawl URL schema with modification handlingconst CRAWL_URL = BASE_URL_SCHEMA.transform(url => modifyCrawlUrl(url));