UniqueSoft
/
firecrawl
mirror of https://github.com/mendableai/firecrawl
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(WebScraper): explicitly ignore 404s

This commit is contained in:
Gergő Móricz 2024-09-23 18:47:07 +02:00
parent 83d8287c14
commit 677faa27f3
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -423,7 +423,7 @@ export async function scrapSingleUrl(
// NOTE: This exception for status codes may only work with fire-engine. In lieu of better error management,
// it's the best we can do. - mogery
if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code "))) {
if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code ") || x === "NOT FOUND")) {
throw new Error(`All scraping methods failed for URL: ${urlToScrap}`);
}