Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(ci): make npm publish survive slow Rekor (sigstore 409 + 30s timeout) (#4323)

Martin Adámeka1508fb1ce86
diff --git a/.github/workflows/publish-to-npm.yml b/.github/workflows/publish-to-npm.yml
index c4fafc856..59f289289 100644
--- a/.github/workflows/publish-to-npm.yml
+++ b/.github/workflows/publish-to-npm.yml
@@ -78,13 +78,14 @@ jobs:
message: "chore: bump ${{ inputs.dist-tag == 'rc' && 'RC' || 'canary' }} versions [skip ci]"
push: false
+ # Rekor outages last minutes; the sigstore patch (30s Rekor timeout + fetch the existing entry on 409) covers the per-call side
- name: Publish to NPM (@latest)
if: inputs.dist-tag == 'prod'
uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
with:
timeout_minutes: 10
max_attempts: 5
- retry_wait_seconds: 30
+ retry_wait_seconds: 60
command: git checkout . && pnpm publish:prod
env:
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
@@ -95,7 +96,7 @@ jobs:
with:
timeout_minutes: 10
max_attempts: 5
- retry_wait_seconds: 30
+ retry_wait_seconds: 60
command: git checkout . && pnpm publish:next
env:
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
@@ -106,7 +107,7 @@ jobs:
with:
timeout_minutes: 10
max_attempts: 5
- retry_wait_seconds: 30
+ retry_wait_seconds: 60
command: git checkout . && pnpm publish:rc
env:
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
diff --git a/patches/[email protected] b/patches/[email protected]
new file mode 100644
index 000000000..f5a8b27d8
--- /dev/null
+++ b/patches/[email protected]
@@ -0,0 +1,22 @@
+diff --git a/dist/config.js b/dist/config.js
+index 373149fe22fb750faeefba5d20a0e432bb91df14..000fbba94c277639c264740599560382f8706399 100644
+--- a/dist/config.js
++++ b/dist/config.js
+@@ -23,7 +23,7 @@ const core_1 = require("@sigstore/core");
+ const sign_1 = require("@sigstore/sign");
+ const verify_1 = require("@sigstore/verify");
+ exports.DEFAULT_RETRY = { retries: 2 };
+-exports.DEFAULT_TIMEOUT = 5000;
++exports.DEFAULT_TIMEOUT = 30000;
+ function createBundleBuilder(bundleType, options) {
+ const bundlerOptions = {
+ signer: initSigner(options),
+@@ -102,7 +102,7 @@ function initWitnesses(options) {
+ witnesses.push(new sign_1.RekorWitness({
+ rekorBaseURL: options.rekorURL,
+ entryType: options.legacyCompatibility ? 'intoto' : 'dsse',
+- fetchOnConflict: false,
++ fetchOnConflict: true,
+ retry: options.retry ?? exports.DEFAULT_RETRY,
+ timeout: options.timeout ?? exports.DEFAULT_TIMEOUT,
+ }));
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index fde795214..1197c2c51 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -182,6 +182,7 @@ overrides:
patchedDependencies:
'@docusaurus/[email protected]': 7e95e0e59f770d24c0e51e1e2635b0a25edf33b38938162fa8c69ab42d38b1fb
'@signalwire/[email protected]': 9c7d64b548a8bb7332cf782e4f959d760e600e78aef2d10562df8a05d7459695
+ [email protected]: 94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17
importers:
@@ -21880,7 +21881,7 @@ snapshots:
npm-registry-fetch: 20.0.1([email protected])
proc-log: 7.0.0
semver: 7.8.5
- sigstore: 5.0.0([email protected])
+ sigstore: 5.0.0(patch_hash=94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17)([email protected])
ssri: 14.0.0
transitivePeerDependencies:
- kerberos
@@ -23485,7 +23486,7 @@ snapshots:
npm-pick-manifest: 12.0.0
npm-registry-fetch: 20.0.1([email protected])
proc-log: 7.0.0
- sigstore: 5.0.0([email protected])
+ sigstore: 5.0.0(patch_hash=94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17)([email protected])
ssri: 14.0.0
tar: 7.5.22
transitivePeerDependencies:
@@ -25185,7 +25186,7 @@ snapshots:
+ [email protected](patch_hash=94d1d48cacdbd6a05f183c4d779c907731d3138f3a29530331b721a2963baf17)([email protected]):
dependencies:
'@sigstore/bundle': 5.0.0
'@sigstore/core': 4.0.1
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 919116570..8af6cebb2 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -84,3 +84,4 @@ publicHoistPattern:
patchedDependencies:
'@docusaurus/[email protected]': patches/@[email protected]
'@signalwire/[email protected]': patches/@[email protected]