Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(ci): make npm publish survive slow Rekor (sigstore 409 + 30s timeout) (#4323)
Martin Adámeka1508fb1ce86
diff --git a/.github/workflows/publish-to-npm.yml b/.github/workflows/publish-to-npm.ymlindex c4fafc856..59f289289 100644--- a/.github/workflows/publish-to-npm.yml+++ b/.github/workflows/publish-to-npm.yml@@ -78,13 +78,14 @@ jobs:message: "chore: bump ${{ inputs.dist-tag == 'rc' && 'RC' || 'canary' }} versions [skip ci]"push: false+ # Rekor outages last minutes; the sigstore patch (30s Rekor timeout + fetch the existing entry on 409) covers the per-call side- name: Publish to NPM (@latest)if: inputs.dist-tag == 'prod'uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0with:timeout_minutes: 10max_attempts: 5- retry_wait_seconds: 30+ retry_wait_seconds: 60command: git checkout . && pnpm publish:prodenv:GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}@@ -95,7 +96,7 @@ jobs:with:timeout_minutes: 10max_attempts: 5- retry_wait_seconds: 30+ retry_wait_seconds: 60command: git checkout . && pnpm publish:nextenv:GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}@@ -106,7 +107,7 @@ jobs:with:timeout_minutes: 10max_attempts: 5- retry_wait_seconds: 30+ retry_wait_seconds: 60command: git checkout . && pnpm publish:rcenv:GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}diff --git a/patches/[email protected] b/patches/[email protected]new file mode 100644index 000000000..f5a8b27d8--- /dev/null+++ b/patches/[email protected]@@ -0,0 +1,22 @@+diff --git a/dist/config.js b/dist/config.js+index 373149fe22fb750faeefba5d20a0e432bb91df14..000fbba94c277639c264740599560382f8706399 100644+--- a/dist/config.js++++ b/dist/config.js+@@ -23,7 +23,7 @@ const core_1 = require("@sigstore/core");+ const sign_1 = require("@sigstore/sign");+ const verify_1 = require("@sigstore/verify");+ exports.DEFAULT_RETRY = { retries: 2 };+-exports.DEFAULT_TIMEOUT = 5000;++exports.DEFAULT_TIMEOUT = 30000;+ function createBundleBuilder(bundleType, options) {+ const bundlerOptions = {+ signer: initSigner(options),+@@ -102,7 +102,7 @@ function initWitnesses(options) {+ witnesses.push(new sign_1.RekorWitness({+ rekorBaseURL: options.rekorURL,+ entryType: options.legacyCompatibility ? 'intoto' : 'dsse',+- fetchOnConflict: false,++ fetchOnConflict: true,+ retry: options.retry ?? exports.DEFAULT_RETRY,+ timeout: options.timeout ?? exports.DEFAULT_TIMEOUT,+ }));diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex fde795214..1197c2c51 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -182,6 +182,7 @@ overrides:patchedDependencies:'@docusaurus/[email protected]': 7e95e0e59f770d24c0e51e1e2635b0a25edf33b38938162fa8c69ab42d38b1fb'@signalwire/[email protected]': 9c7d64b548a8bb7332cf782e4f959d760e600e78aef2d10562df8a05d7459695+ [email protected]: 94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17importers:@@ -21880,7 +21881,7 @@ snapshots:npm-registry-fetch: 20.0.1([email protected])proc-log: 7.0.0semver: 7.8.5- sigstore: 5.0.0([email protected])+ sigstore: 5.0.0(patch_hash=94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17)([email protected])ssri: 14.0.0transitivePeerDependencies:- kerberos@@ -23485,7 +23486,7 @@ snapshots:npm-pick-manifest: 12.0.0npm-registry-fetch: 20.0.1([email protected])proc-log: 7.0.0- sigstore: 5.0.0([email protected])+ sigstore: 5.0.0(patch_hash=94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17)([email protected])ssri: 14.0.0tar: 7.5.22transitivePeerDependencies:@@ -25185,7 +25186,7 @@ snapshots:+ [email protected](patch_hash=94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17)([email protected]):dependencies:'@sigstore/bundle': 5.0.0'@sigstore/core': 4.0.1diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 919116570..8af6cebb2 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -84,3 +84,4 @@ publicHoistPattern:patchedDependencies:'@docusaurus/[email protected]': patches/@[email protected]'@signalwire/[email protected]': patches/@[email protected]+ [email protected]: patches/[email protected]