Merge pull request #485 from mendableai/bugfix/issue-435
[Bug] Issue with crawl going beyond Limit
This commit is contained in:
commit
72eebb0aea
|
@ -420,9 +420,10 @@ export class WebCrawler {
|
|||
".woff",
|
||||
".ttf",
|
||||
".woff2",
|
||||
".webp"
|
||||
".webp",
|
||||
".inc"
|
||||
];
|
||||
return fileExtensions.some((ext) => url.endsWith(ext));
|
||||
return fileExtensions.some((ext) => url.toLowerCase().endsWith(ext));
|
||||
}
|
||||
|
||||
private isSocialMediaOrEmail(url: string): boolean {
|
||||
|
|
Loading…
Reference in New Issue