diff --git a/.github/workflows/fly-direct.yml b/.github/workflows/fly-direct.yml index 944e399..523a262 100644 --- a/.github/workflows/fly-direct.yml +++ b/.github/workflows/fly-direct.yml @@ -29,9 +29,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Change directory - run: cd apps/api - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js + - run: flyctl deploy --remote-only -a firecrawl-scraper-js && curl -X POST https://api.firecrawl.dev/admin/$BULL_AUTH_KEY/unpause + working-directory: ./apps/api env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml index 627409e..16c6e81 100644 --- a/.github/workflows/fly.yml +++ b/.github/workflows/fly.yml @@ -175,12 +175,12 @@ jobs: needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests] steps: - uses: actions/checkout@v3 - - name: Change directory - run: cd apps/api - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js + - run: flyctl deploy --remote-only -a firecrawl-scraper-js && curl -X POST https://api.firecrawl.dev/admin/$BULL_AUTH_KEY/unpause + working-directory: ./apps/api env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} build-and-publish-python-sdk: name: Build and publish Python SDK diff --git a/apps/api/package.json b/apps/api/package.json index 0668f66..ab1615c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -19,7 +19,8 @@ "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest", "mongo-docker-console": "docker exec -it mongodb mongosh", "run-example": "npx ts-node src/example.ts", - "deploy:fly:staging": "fly deploy -c fly.staging.toml" + "deploy:fly": "flyctl deploy && node postdeploy.js https://api.firecrawl.dev", + "deploy:fly:staging": "fly deploy -c fly.staging.toml && node postdeploy.js https://staging-firecrawl-scraper-js.fly.dev" }, "author": "", "license": "ISC", diff --git a/apps/api/postdeploy.js b/apps/api/postdeploy.js new file mode 100644 index 0000000..c1b94d7 --- /dev/null +++ b/apps/api/postdeploy.js @@ -0,0 +1,11 @@ +require("dotenv").config(); + +fetch(process.argv[2] + "/admin/" + process.env.BULL_AUTH_KEY + "/unpause", { + method: "POST" +}).then(async x => { + console.log(await x.text()); + process.exit(0); +}).catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index ba494d8..349e5ed 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -99,6 +99,7 @@ if (cluster.isMaster) { app.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, async (req, res) => { try { const webScraperQueue = getWebScraperQueue(); + const [webScraperActive] = await Promise.all([ webScraperQueue.getActiveCount(), ]); @@ -153,6 +154,11 @@ if (cluster.isMaster) { } }); + app.post(`/admin/${process.env.BULL_AUTH_KEY}/unpause`, async (req, res) => { + await getWebScraperQueue().resume(true); + res.json({ ok: true }); + }); + app.get(`/serverHealthCheck`, async (req, res) => { try { const webScraperQueue = getWebScraperQueue(); @@ -273,11 +279,4 @@ if (cluster.isMaster) { }); console.log(`Worker ${process.pid} started`); - - (async () => { - const wsq = getWebScraperQueue(); - if (await wsq.isPaused(false)) { - await wsq.resume(false); - } - })(); }