From f0695c712307b06bde55e251f799373882b6a7ad Mon Sep 17 00:00:00 2001
From: Nicolas <nicolascamara29@gmail.com>
Date: Tue, 23 Apr 2024 17:04:10 -0700
Subject: [PATCH] Update single_url.ts

---
 apps/api/src/scraper/WebScraper/single_url.ts | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index fcbb688..e110b0e 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -23,13 +23,14 @@ export async function scrapWithCustomFirecrawl(
 
 export async function scrapWithScrapingBee(
   url: string,
-  wait_browser: string = "domcontentloaded"
+  wait_browser: string = "domcontentloaded",
+  timeout: number = 15000
 ): Promise<string> {
   try {
     const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY);
     const response = await client.get({
       url: url,
-      params: { timeout: 15000, wait_browser: wait_browser },
+      params: { timeout: timeout, wait_browser: wait_browser },
       headers: { "ScrapingService-Request": "TRUE" },
     });
 
@@ -106,11 +107,11 @@ export async function scrapSingleUrl(
     let text = "";
     switch (method) {
       case "firecrawl-scraper":
-        text = await scrapWithCustomFirecrawl(url);
+        text = await scrapWithCustomFirecrawl(url,);
         break;
       case "scrapingBee":
         if (process.env.SCRAPING_BEE_API_KEY) {
-          text = await scrapWithScrapingBee(url);
+          text = await scrapWithScrapingBee(url,"domcontentloaded", pageOptions.fallback  === false? 7000 : 15000);
         }
         break;
       case "playwright":