From 3f138e559e3c33f09f817f84eb900b22a150e745 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 24 Sep 2024 15:14:26 -0400 Subject: [PATCH] Update website_params.ts --- .../WebScraper/utils/custom/website_params.ts | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 8169d9d..659e4b8 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -49,26 +49,6 @@ export const urlSpecificParams = { "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", }, }, - "ycombinator.com":{ - defaultScraper: "fire-engine", - params: { - wait_browser: "networkidle2", - block_resources: false, - wait: 3000, - }, - headers: { - "User-Agent": - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "sec-fetch-site": "same-origin", - "sec-fetch-mode": "cors", - "sec-fetch-dest": "empty", - referer: "https://www.google.com/", - "accept-language": "en-US,en;q=0.9", - "accept-encoding": "gzip, deflate, br", - accept: - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", - }, - }, "developers.notion.com":{ defaultScraper: "fire-engine", params: {