UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update single_url.ts

This commit is contained in:
Nicolas 2024-04-28 12:44:00 -07:00
parent d8ee4e90d6
commit 68838c9e0d
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -20,10 +20,15 @@ export async function generateRequestParams(
headers: { "ScrapingService-Request": "TRUE" },
};
const urlKey = new URL(url).hostname;
if (urlSpecificParams.hasOwnProperty(urlKey)) {
return { ...defaultParams, ...urlSpecificParams[urlKey] };
} else {
try {
const urlKey = new URL(url).hostname;
if (urlSpecificParams.hasOwnProperty(urlKey)) {
return { ...defaultParams, ...urlSpecificParams[urlKey] };
} else {
return defaultParams;
}
} catch (error) {
console.error(`Error generating URL key: ${error}`);
return defaultParams;
}
}