From 1742e4ceaeb322bb42c1162a8b8f3050c5e09406 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 2 Aug 2024 19:25:15 -0400 Subject: [PATCH] Nick: --- apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts | 3 ++- apps/api/src/scraper/WebScraper/utils/custom/website_params.ts | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index 0f4c232..ba67043 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -47,7 +47,7 @@ export async function scrapWithFireEngine({ try { const reqParams = await generateRequestParams(url); const waitParam = reqParams["params"]?.wait ?? waitFor; - const engineParam = reqParams["params"]?.engine ?? fireEngineOptions?.engine ?? "playwright"; + const engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "playwright"; const screenshotParam = reqParams["params"]?.screenshot ?? screenshot; const fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions; @@ -64,6 +64,7 @@ export async function scrapWithFireEngine({ `⛏️ Fire-Engine (${engine}): Scraping ${url} | params: { wait: ${waitParam}, screenshot: ${screenshotParam}, method: ${fireEngineOptionsParam?.method ?? "null"} }` ); + const response = await axios.post( process.env.FIRE_ENGINE_BETA_URL + endpoint, { diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 02e3e2e..8583e61 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -236,8 +236,6 @@ export const urlSpecificParams = { defaultScraper: "fire-engine", params:{ fireEngineOptions:{ - mobileProxy: true, - method: "get", engine: "chrome-cdp", }, },