Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): accept `UInt8Array` in `KVS.setValue()` (#2682)
Martin Adámek8ef0e60ca6fb
diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex 7574d264a..0f779dd2b 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -20,7 +20,7 @@ const requestOptionalPredicates = {loadedUrl: ow.optional.string.url,uniqueKey: ow.optional.string,method: ow.optional.string,- payload: ow.optional.any(ow.string, ow.buffer),+ payload: ow.optional.any(ow.string, ow.uint8Array),noRetry: ow.optional.boolean,retryCount: ow.optional.number,sessionRotationCount: ow.optional.number,diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.tsindex fac7535a6..b3ed3d353 100644--- a/packages/core/src/serialization.ts+++ b/packages/core/src/serialization.ts@@ -79,8 +79,8 @@ export async function serializeArray<T>(data: T[]): Promise<Buffer> {* when apify-client supports streams.* @internal*/-export async function deserializeArray<T extends string | Buffer>(compressedData: Buffer): Promise<T[]> {- ow(compressedData, ow.buffer);+export async function deserializeArray<T extends string | Buffer>(compressedData: Buffer | Uint8Array): Promise<T[]> {+ ow(compressedData, ow.uint8Array);const { chunks, collector } = createChunkCollector<T>({ fromValuesStream: true });await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);@@ -96,8 +96,8 @@ export async function deserializeArray<T extends string | Buffer>(compressedData* optimized to ingest a Stream if and when apify-client supports streams.* @internal*/-export function createDeserialize(compressedData: Buffer): Readable {- ow(compressedData, ow.buffer);+export function createDeserialize(compressedData: Buffer | Uint8Array): Readable {+ ow(compressedData, ow.uint8Array);const streamArray = StreamArray.withParser();const destination = pluckValue(streamArray);diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex dbbe1aeb7..59dfd3bdb 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -336,7 +336,7 @@ export class KeyValueStore {if (options.contentType &&!(- ow.isValid(value, ow.any(ow.string, ow.buffer)) ||+ ow.isValid(value, ow.any(ow.string, ow.uint8Array)) ||(ow.isValid(value, ow.object) && typeof (value as Dictionary).pipe === 'function'))) {diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 89b2a0d56..775aa9f5a 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -190,7 +190,7 @@ export async function gotoExtended(url: ow.string.url,method: ow.optional.string,headers: ow.optional.object,- payload: ow.optional.any(ow.string, ow.buffer),+ payload: ow.optional.any(ow.string, ow.uint8Array),}),);ow(gotoOptions, ow.object);diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex db53562b1..390e1fc1d 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -465,7 +465,7 @@ export async function gotoExtended(url: ow.string.url,method: ow.optional.string,headers: ow.optional.object,- payload: ow.optional.any(ow.string, ow.buffer),+ payload: ow.optional.any(ow.string, ow.uint8Array),}),);ow(gotoOptions, ow.object);