Merge branch 'main' of https://github.com/mendableai/firecrawl

2024-09-07 13:12:52 -03:00 · 2024-09-07 13:12:52 -03:00 · 32097fa29d
parent 5758af3291 b3e2ca9a11
commit 32097fa29d
1 changed files with 0 additions and 8 deletions
--- a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts
+++ b/apps/api/src/scraper/WebScraper/utils/excludeTags.ts
@ -39,16 +39,8 @@ export const excludeNonMainTags = [
  "#search",
  ".share",
  "#share",
-  ".pagination",
-  "#pagination",
  ".widget",
  "#widget",
-  ".related",
-  "#related",
-  ".tag",
-  "#tag",
-  ".category",
-  "#category",
  ".cookie",
  "#cookie"
 ];