diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index 5ee8cda..60dc301 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -164,7 +164,7 @@ export class WebCrawler { concurrencyLimit, inProgress ); - + if ( urls.length === 0 && this.filterLinks([this.initialUrl], limit, this.maxCrawledDepth).length > 0 @@ -420,9 +420,10 @@ export class WebCrawler { ".woff", ".ttf", ".woff2", - ".webp" + ".webp", + ".inc" ]; - return fileExtensions.some((ext) => url.endsWith(ext)); + return fileExtensions.some((ext) => url.toLowerCase().endsWith(ext)); } private isSocialMediaOrEmail(url: string): boolean {