From 43730b5db6b78433073dfc7de8ea2a2bef365cd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 24 Sep 2024 20:03:49 +0200
Subject: [PATCH] feat(WebScraper): always report error of last scraper in
 order

---
 apps/api/src/scraper/WebScraper/single_url.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index 80491f3..66ae744 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -392,8 +392,12 @@ export async function scrapSingleUrl(
       if (attempt.pageStatusCode) {
         pageStatusCode = attempt.pageStatusCode;
       }
-      if (attempt.pageError && attempt.pageStatusCode >= 400) {
+      if (attempt.pageError && (attempt.pageStatusCode >= 400 || scrapersInOrder.indexOf(scraper) === scrapersInOrder.length - 1)) { // force pageError if it's the last scraper and it failed too
         pageError = attempt.pageError;
+        
+        if (attempt.pageStatusCode < 400 || !attempt.pageStatusCode) {
+          pageStatusCode = 500;
+        }
       } else if (attempt && attempt.pageStatusCode && attempt.pageStatusCode < 400) {
         pageError = undefined;
       }