feat: unpause by http request

.github/workflows/fly-direct.yml

@@ -29,9 +29,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Change directory
-        run: cd apps/api
       - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js
+      - run: flyctl deploy --remote-only -a firecrawl-scraper-js && curl -X POST https://api.firecrawl.dev/admin/$BULL_AUTH_KEY/unpause
+        working-directory: ./apps/api
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+          BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}

.github/workflows/fly.yml

@@ -175,12 +175,12 @@ jobs:
     needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests]
     steps:
       - uses: actions/checkout@v3
-      - name: Change directory
-        run: cd apps/api
       - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js
+      - run: flyctl deploy --remote-only -a firecrawl-scraper-js && curl -X POST https://api.firecrawl.dev/admin/$BULL_AUTH_KEY/unpause
+        working-directory: ./apps/api
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+          BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
 
   build-and-publish-python-sdk:
     name: Build and publish Python SDK

apps/api/package.json

@@ -19,7 +19,8 @@
     "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
     "mongo-docker-console": "docker exec -it mongodb mongosh",
     "run-example": "npx ts-node src/example.ts",
-    "deploy:fly:staging": "fly deploy -c fly.staging.toml"
+    "deploy:fly": "flyctl deploy && node postdeploy.js https://api.firecrawl.dev",
+    "deploy:fly:staging": "fly deploy -c fly.staging.toml && node postdeploy.js https://staging-firecrawl-scraper-js.fly.dev"
   },
   "author": "",
   "license": "ISC",

apps/api/postdeploy.js

@@ -0,0 +1,11 @@
+require("dotenv").config();
+
+fetch(process.argv[2] + "/admin/" + process.env.BULL_AUTH_KEY + "/unpause", {
+    method: "POST"
+}).then(async x => {
+    console.log(await x.text());
+    process.exit(0);
+}).catch(e => {
+    console.error(e);
+    process.exit(1);
+});

apps/api/src/index.ts

@@ -99,6 +99,7 @@ if (cluster.isMaster) {
   app.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, async (req, res) => {
     try {
       const webScraperQueue = getWebScraperQueue();
+
       const [webScraperActive] = await Promise.all([
         webScraperQueue.getActiveCount(),
       ]);
@@ -153,6 +154,11 @@ if (cluster.isMaster) {
     }
   });
 
+  app.post(`/admin/${process.env.BULL_AUTH_KEY}/unpause`, async (req, res) => {
+    await getWebScraperQueue().resume(true);
+    res.json({ ok: true });
+  });
+
   app.get(`/serverHealthCheck`, async (req, res) => {
     try {
       const webScraperQueue = getWebScraperQueue();
@@ -273,11 +279,4 @@ if (cluster.isMaster) {
   });
 
   console.log(`Worker ${process.pid} started`);
-
-  (async () => {
-    const wsq = getWebScraperQueue();
-    if (await wsq.isPaused(false)) {
-      await wsq.resume(false);
-    }
-  })();
 }