From 90b7fff3665649c4714c5d0195b8121e524308cb Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 24 Jun 2024 16:52:01 -0300 Subject: [PATCH] Update crawler.ts --- apps/api/src/scraper/WebScraper/crawler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index 2e2dec2..5003845 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -418,7 +418,7 @@ export class WebCrawler { const normalizedUrl = normalizeUrl(url); const normalizedSitemapLinks = sitemapLinks.map(link => normalizeUrl(link)); - + // has to be greater than 0 to avoid adding the initial URL to the sitemap links, and preventing crawler to crawl if (!normalizedSitemapLinks.includes(normalizedUrl) && sitemapLinks.length > 0) { sitemapLinks.push(url); }