UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update parameters for platform.openai.com

This commit is contained in:
Thomas Kosmas 2024-08-12 22:49:28 +03:00
parent 9d187bf3bb
commit 98be29c963
1 changed files with 6 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
View File

@ -1,24 +1,11 @@
export const urlSpecificParams = { export const urlSpecificParams = {
"platform.openai.com": { "platform.openai.com": {
params: { defaultScraper: "fire-engine",
wait_browser: "networkidle2", params:{
block_resources: false, wait: 3000,
}, fireEngineOptions:{
headers: { engine: "chrome-cdp"
"User-Agent": },
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"sec-fetch-site": "same-origin",
"sec-fetch-mode": "cors",
"sec-fetch-dest": "empty",
referer: "https://www.google.com/",
"accept-language": "en-US,en;q=0.9",
"accept-encoding": "gzip, deflate, br",
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
},
cookies: {
__cf_bm:
"mC1On8P2GWT3A5UeSYH6z_MP94xcTAdZ5jfNi9IT2U0-1714327136-1.0.1.1-ILAP5pSX_Oo9PPo2iHEYCYX.p9a0yRBNLr58GHyrzYNDJ537xYpG50MXxUYVdfrD.h3FV5O7oMlRKGA0scbxaQ",
}, },
}, },
"support.greenpay.me":{ "support.greenpay.me":{