Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Updated publish.sh to allow beta deploy from other branches, enabled aws cloudfront invalidation
Jan Curn9b32347384f7
diff --git a/publish.sh b/publish.shindex f4fcd27cb..e3960a568 100755--- a/publish.sh+++ b/publish.sh@@ -29,7 +29,7 @@ aws s3 cp "${DOC_DIR}/" "s3://${AWS_BUCKET}/${GIT_TAG}/" --recursive --region usecho "Pushing to git ..."git push-# Master gets published as LATEST if that version doesn't exists yet and retagged as LATEST otherwise.+# Master gets published as LATEST - the package already needs to be published as BETA.if [ "${BRANCH}" = "master" ]; thenEXISTING_NPM_VERSION=$(npm view ${PACKAGE_NAME} versions | grep ${PACKAGE_VERSION} | tee) # Using tee to swallow non-zero exit codeif [ -z "${EXISTING_NPM_VERSION}" ]; then@@ -40,11 +40,11 @@ if [ "${BRANCH}" = "master" ]; thenRUNNING_FROM_SCRIPT=1 npm dist-tag add ${PACKAGE_NAME}@${PACKAGE_VERSION} latestecho "Copy doc to latest folder..."aws s3 cp "s3://${AWS_BUCKET}/${GIT_TAG}/" "s3://${AWS_BUCKET}/latest/" --recursive --region us-east-1 --acl public-read --cache-control "public, max-age=3600"- #aws cloudfront create-invalidation --distribution-id E29XCV9LE9131X --paths "/docs/sdk/apify-runtime-js/*"+ aws cloudfront create-invalidation --distribution-id E29XCV9LE9131X --paths "/docs/sdk/apify-runtime-js/*"fi-# Develop branch gets published as BETA and we don't allow to override tag of existing version.-elif [ "${BRANCH}" = "develop" ]; then+# Any other branch gets published as BETA and we don't allow to override tag of existing version.+elseecho "Publishing version ${PACKAGE_VERSION} with tag \"beta\" ..."RUNNING_FROM_SCRIPT=1 npm publish --tag beta@@ -55,12 +55,8 @@ elif [ "${BRANCH}" = "develop" ]; thenecho "Copy docs to S3 to beta folder..."aws s3 cp "s3://${AWS_BUCKET}/${GIT_TAG}/" "s3://${AWS_BUCKET}/beta/" --recursive --region us-east-1 --acl public-read --cache-control "public, max-age=3600"- #aws cloudfront create-invalidation --distribution-id E29XCV9LE9131X --paths "/docs/sdk/apify-runtime-js/*"+ aws cloudfront create-invalidation --distribution-id E29XCV9LE9131X --paths "/docs/sdk/apify-runtime-js/*"-# For other branch throw an error.-else- printf "${RED}You can publish from develop and master branches only!${NC}\n"- exit 1fi