fix(WebScraper): explicitly ignore 404s
This commit is contained in:
parent
83d8287c14
commit
677faa27f3
|
@ -423,7 +423,7 @@ export async function scrapSingleUrl(
|
||||||
|
|
||||||
// NOTE: This exception for status codes may only work with fire-engine. In lieu of better error management,
|
// NOTE: This exception for status codes may only work with fire-engine. In lieu of better error management,
|
||||||
// it's the best we can do. - mogery
|
// it's the best we can do. - mogery
|
||||||
if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code "))) {
|
if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code ") || x === "NOT FOUND")) {
|
||||||
throw new Error(`All scraping methods failed for URL: ${urlToScrap}`);
|
throw new Error(`All scraping methods failed for URL: ${urlToScrap}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue