Update handleCustomScraping.ts

2024-06-04 12:22:46 -07:00 · 2024-06-04 12:22:46 -07:00 · 96257b7b17
parent 674500affa
commit 96257b7b17
1 changed files with 1 additions and 0 deletions
--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
+++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
@ -14,6 +14,7 @@ export async function handleCustomScraping(
    };
  }

+  // Check for Vanta security portals
  if (text.includes('<link href="https://static.vanta.com')) {
    console.log(
      `Vanta link detected for ${url}, using Fire Engine with wait time 3000ms`