From f48ff36b32978fb4d0c54dd7efcbaad5f22e9a5d Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Wed, 31 Jul 2024 09:28:43 -0300
Subject: [PATCH] added .inc files and forced lower case comparison

---
 apps/api/src/scraper/WebScraper/crawler.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 5ee8cda..60dc301 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -164,7 +164,7 @@ export class WebCrawler {
       concurrencyLimit,
       inProgress
     );
-   
+    
     if (
       urls.length === 0 &&
       this.filterLinks([this.initialUrl], limit, this.maxCrawledDepth).length > 0
@@ -420,9 +420,10 @@ export class WebCrawler {
       ".woff",
       ".ttf",
       ".woff2",
-      ".webp"
+      ".webp",
+      ".inc"
     ];
-    return fileExtensions.some((ext) => url.endsWith(ext));
+    return fileExtensions.some((ext) => url.toLowerCase().endsWith(ext));
   }
 
   private isSocialMediaOrEmail(url: string): boolean {