Merge pull request #449 from mendableai/bugfix/malformed-url-sitemap

Added regex for links in sitemap

apps/api/src/scraper/WebScraper/crawler.ts

@@ -64,7 +64,7 @@ export class WebCrawler {
   private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
     return sitemapLinks
       .filter((link) => {
-        const url = new URL(link);
+        const url = new URL(link.trim(), this.baseUrl);
         const path = url.pathname;
         
         const depth = getURLDepth(url.toString());