Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas30a375d693e2
diff --git a/.github/workflows/fly-direct.yml b/.github/workflows/fly-direct.ymldeleted file mode 100644index 3ee460e68..000000000--- a/.github/workflows/fly-direct.yml+++ /dev/null@@ -1,46 +0,0 @@-name: Fly Deploy Direct-on:- schedule:- - cron: '0 * * * *'--env:- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}- HOST: ${{ secrets.HOST }}- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}- LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}- POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}- POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}- PORT: ${{ secrets.PORT }}- REDIS_URL: ${{ secrets.REDIS_URL }}- SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}- PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}- CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}- SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}- USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }}- ENV: ${{ secrets.ENV }}--jobs:- deploy:- name: Deploy app- runs-on: ubuntu-latest- timeout-minutes: 15- steps:- - uses: actions/checkout@v3- - uses: superfly/flyctl-actions/setup-flyctl@master- - run: flyctl deploy --remote-only -a firecrawl-scraper-js --build-secret SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN- working-directory: ./apps/api- env:- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.ymldeleted file mode 100644index ed8dade17..000000000--- a/.github/workflows/fly.yml+++ /dev/null@@ -1,81 +0,0 @@-name: Fly Deploy-on:- push:- branches:- - main--env:- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}- HOST: ${{ secrets.HOST }}- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}- LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}- POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}- POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}- PORT: ${{ secrets.PORT }}- REDIS_URL: ${{ secrets.REDIS_URL }}- SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}- PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}- CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}- SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}- USE_DB_AUTHENTICATION: ${{ secrets.USE_DB_AUTHENTICATION }}- ENV: ${{ secrets.ENV }}--jobs:- pre-deploy:- name: Pre-deploy checks- runs-on: ubuntu-latest- services:- redis:- image: redis- ports:- - 6379:6379- steps:- - uses: actions/checkout@v3- - name: Set up Node.js- uses: actions/setup-node@v3- with:- node-version: "20"- - name: Install pnpm- run: npm install -g pnpm- - name: Install dependencies- run: pnpm install- working-directory: ./apps/api- - name: Start the application- run: npm start &- working-directory: ./apps/api- id: start_app- - name: Start workers- run: npm run workers &- working-directory: ./apps/api- id: start_workers- - name: Run E2E tests- run: |- npm run test:prod- working-directory: ./apps/api-- deploy:- name: Deploy app- needs: pre-deploy- runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v3- - uses: superfly/flyctl-actions/setup-flyctl@master- - run: flyctl deploy --remote-only -a firecrawl-scraper-js --build-secret SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN- working-directory: ./apps/api- env:- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}--\ No newline at end of file