chore(release): 2.0.7

Add error message, if returned search HTML does not contain required elements

CHANGELOG.md

@@ -2,6 +2,34 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [2.0.7](https://github.com/towfiqi/serpbear/compare/v2.0.6...v2.0.7) (2025-02-23)
+
+
+### Bug Fixes
+
+* Resolves AdWords integration issue. ([36ed4cf](https://github.com/towfiqi/serpbear/commit/36ed4cf800c1fd0e3df4e807faaa1fdb863df5e4))
+* resolves broken CDN images. ([bf911b4](https://github.com/towfiqi/serpbear/commit/bf911b4e45b9007a05ce6399838da3276161c61d))
+
+### [2.0.6](https://github.com/towfiqi/serpbear/compare/v2.0.5...v2.0.6) (2024-11-15)
+
+
+### Bug Fixes
+
+* Ensures Docker build uses matching npm package versions from package.json ([4fef1a9](https://github.com/towfiqi/serpbear/commit/4fef1a9abc737da67ab1ea0c4efce8194890545e))
+* Resolves broken Docker build due to croner package version mismatch. ([aeed1f8](https://github.com/towfiqi/serpbear/commit/aeed1f8559e044bf658d930a22fa91f38cfedc6b)), closes [#247](https://github.com/towfiqi/serpbear/issues/247)
+* Resolves broken Proxy Scraper functionality. ([649f412](https://github.com/towfiqi/serpbear/commit/649f412303dd50127b3736740962863f735f76eb)), closes [#248](https://github.com/towfiqi/serpbear/issues/248)
+* Resolves Google Ads search volume data loading issue. ([12eac2b](https://github.com/towfiqi/serpbear/commit/12eac2b01235e9eae06882d6a2c50c793b890661))
+
+### [2.0.5](https://github.com/towfiqi/serpbear/compare/v2.0.4...v2.0.5) (2024-11-12)
+
+
+### Bug Fixes
+
+* Fixes broken scrape result issue for keywords with special characters. ([d9d7c63](https://github.com/towfiqi/serpbear/commit/d9d7c6347e99e35f1e48f20b1e8f999612d69a72)), closes [#221](https://github.com/towfiqi/serpbear/issues/221)
+* Fixes misaligned Keywords table UI content. ([faa3519](https://github.com/towfiqi/serpbear/commit/faa3519a29fc61ef8bd2ce9275a6674f1c7946e0))
+* Resolves "Add Domain" UI confusion. ([17fb2c4](https://github.com/towfiqi/serpbear/commit/17fb2c40cc6b57ab2fe6aeb940dececd1a83411f))
+* Resolves broken Scrapingrobot scraper on new installs. ([bc02c92](https://github.com/towfiqi/serpbear/commit/bc02c929ba7efd6b4b6a09495af7310c155a26bd)), closes [#243](https://github.com/towfiqi/serpbear/issues/243)
+
 ### [2.0.4](https://github.com/towfiqi/serpbear/compare/v2.0.3...v2.0.4) (2024-11-10)
 
 

Dockerfile

@@ -40,7 +40,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc
 COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv croner @googleapis/searchconsole sequelize-cli @isaacs/ttlcache
+RUN npm i cryptr@6.0.3 dotenv@16.0.3 croner@9.0.0 @googleapis/searchconsole@1.0.5 sequelize-cli@6.6.2 @isaacs/ttlcache@1.4.1
 RUN npm i -g concurrently
 
 USER nextjs

README.md

@@ -2,13 +2,13 @@
 
 # SerpBear
 
-![Codacy Badge](https://app.codacy.com/project/badge/Grade/7e7a0030c3f84c6fb56a3ce6273fbc1d) ![GitHub](https://img.shields.io/github/license/towfiqi/serpbear) ![GitHub package.json version](https://img.shields.io/github/package-json/v/towfiqi/serpbear) ![Docker Pulls](https://img.shields.io/docker/pulls/towfiqi/serpbear)
+![Codacy Badge](https://app.codacy.com/project/badge/Grade/7e7a0030c3f84c6fb56a3ce6273fbc1d) ![GitHub](https://img.shields.io/github/license/towfiqi/serpbear) ![GitHub package.json version](https://img.shields.io/github/package-json/v/towfiqi/serpbear) ![Docker Pulls](https://img.shields.io/docker/pulls/towfiqi/serpbear) [![StandWithPalestine](https://raw.githubusercontent.com/Safouene1/support-palestine-banner/master/StandWithPalestine.svg)](https://www.youtube.com/watch?v=bjtDsd0g468&rco=1)
 
 #### [Documentation](https://docs.serpbear.com/) | [Changelog](https://github.com/towfiqi/serpbear/blob/main/CHANGELOG.md) | [Docker Image](https://hub.docker.com/r/towfiqi/serpbear)
 
 SerpBear is an Open Source Search Engine Position Tracking and Keyword Research App. It allows you to track your website's keyword positions in Google and get notified of their position change.
 
-![Easy to Use Search Engine Rank Tracker](https://erevanto.sirv.com/Images/serpbear/serpbear_readme_v2.gif)
+![Easy to Use Search Engine Rank Tracker](https://serpbear.b-cdn.net/serpbear_readme_v2.gif)
 
 #### Features
 

__mocks__/utils.tsx

@@ -1,22 +1,26 @@
 import { render } from '@testing-library/react';
-import { rest } from 'msw';
+import { http } from 'msw';
 import * as React from 'react';
 import { QueryClient, QueryClientProvider } from 'react-query';
 
 export const handlers = [
-    rest.get(
+    http.get(
         '*/react-query',
-        (req, res, ctx) => {
+        ({ request, params }) => {
-            return res(
+            return new Response(
-                ctx.status(200),
+                JSON.stringify({
-                ctx.json({
                     name: 'mocked-react-query',
                 }),
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                },
             );
         },
     ),
 ];
-
 const createTestQueryClient = () => new QueryClient({
     defaultOptions: {
         queries: {

components/domains/AddDomain.tsx

@@ -36,7 +36,7 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
         }
       });
       if (invalidDomains.length > 0) {
-         setNewDomainError(`Please Insert Valid Domain URL. Invalid URLs: ${invalidDomains.join(', ')}`);
+         setNewDomainError(`Please Insert Valid Website URL. ${invalidDomains.length > 1 ? `Invalid URLs: ${invalidDomains.join(', ')}` : ''}`);
       } else if (domainsTobeAdded.length > 0) {
             console.log('domainsTobeAdded :', domainsTobeAdded);
          addMutate(domainsTobeAdded);
@@ -51,11 +51,11 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
    return (
       <Modal closeModal={() => { closeModal(false); }} title={'Add New Domain'}>
          <div data-testid="adddomain_modal">
-            <h4 className='text-sm mt-4'>Domain URL</h4>
+            <h4 className='text-sm mt-4 pb-2'>Website URL(s)</h4>
             <textarea
                className={`w-full h-40 border rounded border-gray-200 p-4 outline-none
                 focus:border-indigo-300 ${newDomainError ? ' border-red-400 focus:border-red-400' : ''}`}
-               placeholder="Type or Paste URLs here. Insert Each URL in a New line."
+               placeholder={'Type or Paste URLs here. Insert Each URL in a New line. eg: \nhttps://mysite.com/ \nhttps://anothersite.com/ '}
                value={newDomain}
                autoFocus={true}
                onChange={handleDomainInput}>

components/keywords/Keyword.tsx

@@ -97,7 +97,7 @@ const Keyword = (props: KeywordProps) => {
       className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
       lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
 
-         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
+         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-28 lg:w-auto lg:flex lg:items-center'>
             <button
                className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
                ${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}

cron.js

@@ -1,7 +1,8 @@
+/* eslint-disable no-new */
 const Cryptr = require('cryptr');
 const { promises } = require('fs');
 const { readFile } = require('fs');
-const Cron = require('croner');
+const { Cron } = require('croner');
 require('dotenv').config({ path: './.env.local' });
 
 const getAppSettings = async () => {
@@ -71,7 +72,7 @@ const runAppCronJobs = () => {
       const scrape_interval = settings.scrape_interval || 'daily';
       if (scrape_interval !== 'never') {
          const scrapeCronTime = generateCronTime(scrape_interval);
-         Cron(scrapeCronTime, () => {
+         new Cron(scrapeCronTime, () => {
             // console.log('### Running Keyword Position Cron Job!');
             const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
             fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
@@ -89,7 +90,7 @@ const runAppCronJobs = () => {
       if (notif_interval) {
          const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
          if (cronTime) {
-            Cron(cronTime, () => {
+            new Cron(cronTime, () => {
                // console.log('### Sending Notification Email...');
                const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
                fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
@@ -106,7 +107,7 @@ const runAppCronJobs = () => {
 
    // Run Failed scraping CRON (Every Hour)
    const failedCronTime = generateCronTime('hourly');
-   Cron(failedCronTime, () => {
+   new Cron(failedCronTime, () => {
       // console.log('### Retrying Failed Scrapes...');
 
       readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
@@ -135,7 +136,7 @@ const runAppCronJobs = () => {
    // Run Google Search Console Scraper Daily
    if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
       const searchConsoleCRONTime = generateCronTime('daily');
-      Cron(searchConsoleCRONTime, () => {
+      new Cron(searchConsoleCRONTime, () => {
          const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
          fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts)
          .then((res) => res.json())

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "2.0.4",
+  "version": "2.0.7",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -18,16 +18,16 @@
     "release": "standard-version"
   },
   "dependencies": {
-    "@googleapis/searchconsole": "^1.0.0",
+    "@googleapis/searchconsole": "^1.0.5",
     "@isaacs/ttlcache": "^1.4.1",
     "@types/react-transition-group": "^4.4.5",
-    "axios": "^1.1.3",
+    "axios": "^1.7.7",
     "axios-retry": "^3.3.1",
     "chart.js": "^3.9.1",
-    "cheerio": "^1.0.0-rc.12",
+    "cheerio": "^1.0.0",
     "concurrently": "^7.6.0",
     "cookies": "^0.8.0",
-    "croner": "^5.3.5",
+    "croner": "^9.0.0",
     "cryptr": "^6.0.3",
     "dayjs": "^1.11.5",
     "dotenv": "^16.0.3",
@@ -35,7 +35,6 @@
     "https-proxy-agent": "^5.0.1",
     "isomorphic-fetch": "^3.0.0",
     "jsonwebtoken": "^9.0.2",
-    "msw": "^0.49.0",
     "next": "^12.3.4",
     "nodemailer": "^6.9.9",
     "react": "18.2.0",
@@ -50,11 +49,11 @@
     "reflect-metadata": "^0.1.13",
     "sequelize": "^6.34.0",
     "sequelize-typescript": "^2.1.6",
-    "sqlite3": "^5.1.6",
+    "sqlite3": "^5.1.7",
-    "umzug": "^3.6.1"
+    "umzug": "^3.8.2"
   },
   "devDependencies": {
-    "@testing-library/jest-dom": "^6.1.4",
+    "@testing-library/jest-dom": "^6.6.3",
     "@testing-library/react": "^14.0.0",
     "@types/cookies": "^0.7.7",
     "@types/cryptr": "^4.0.1",
@@ -74,15 +73,16 @@
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "jest-fetch-mock": "^3.0.3",
+    "msw": "^2.6.4",
     "next-router-mock": "^0.9.10",
-    "postcss": "^8.4.31",
+    "postcss": "^8.4.49",
     "prettier": "^2.7.1",
     "resize-observer-polyfill": "^1.5.1",
-    "sass": "^1.55.0",
+    "sass": "^1.80.7",
     "sequelize-cli": "^6.6.2",
     "standard-version": "^9.5.0",
     "stylelint-config-standard": "^29.0.0",
-    "tailwindcss": "^3.1.8",
+    "tailwindcss": "^3.4.14",
-    "typescript": "4.8.4"
+    "typescript": "^4.8.4"
   }
 }

pages/api/volume.ts

@@ -36,14 +36,13 @@ const updatekeywordVolume = async (req: NextApiRequest, res: NextApiResponse<Key
          keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
       }
       if (domain) {
-         // const allDomain = domain === 'all';
+         const allDomain = domain === 'all';
-         // const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
+         const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
-         // keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
+         keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
       }
 
       if (keywordsToSend.length > 0) {
          const keywordsVolumeData = await getKeywordsVolume(keywordsToSend);
-         // console.log('keywordsVolumeData :', keywordsVolumeData);
          if (keywordsVolumeData.error) {
             return res.status(400).json({ keywords: [], error: keywordsVolumeData.error });
          }

scrapers/services/hasdata.ts

@@ -20,11 +20,10 @@ const hasdata:ScraperSettings = {
    scrapeURL: (keyword, settings) => {
       const country = keyword.country || 'US';
       const countryName = countries[country][0];
-      const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
-      return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
+      return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURIComponent(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
    },
    resultObjectKey: 'organicResults',
-   
    serpExtractor: (content) => {
       const extractedResult = [];
       const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[];

scrapers/services/proxy.ts

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 const proxy:ScraperSettings = {
    id: 'proxy',
@@ -16,6 +16,13 @@ const proxy:ScraperSettings = {
 
       const $ = cheerio.load(content);
       let lastPosition = 0;
+      const hasValidContent = $('body').find('#main');
+      if (hasValidContent.length === 0) {
+         const msg = '[ERROR] Scraped search results from proxy do not adhere to expected format. Unable to parse results';
+         console.log(msg);
+         throw new Error(msg);
+      }
+
       const mainContent = $('body').find('#main');
       const children = $(mainContent).find('h3');
 

scrapers/services/searchapi.ts

@@ -20,8 +20,8 @@ const searchapi:ScraperSettings = {
   scrapeURL: (keyword) => {
    const country = keyword.country || 'US';
    const countryName = countries[country][0];
-   const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+   const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
-     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
+     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
   },
   resultObjectKey: 'organic_results',
   serpExtractor: (content) => {

scrapers/services/serpapi.ts

@@ -19,8 +19,8 @@ const serpapi:ScraperSettings = {
    },
    scrapeURL: (keyword, settings) => {
       const countryName = countries[keyword.country || 'US'][0];
-      const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const location = keyword.city && keyword.country ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
-      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
+      return `https://serpapi.com/search?q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

scrapers/services/serper.ts

@@ -12,7 +12,8 @@ const serper:ScraperSettings = {
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
       const lang = countryData[country][2];
-      return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
+      console.log('Serper URL :', `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`);
+      return `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
    },
    resultObjectKey: 'organic',
    serpExtractor: (content) => {

scrapers/services/serply.ts

@@ -20,7 +20,7 @@ const serply:ScraperSettings = {
    },
    scrapeURL: (keyword) => {
       const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';
-      return `https://api.serply.io/v1/search/q=${encodeURI(keyword.keyword)}&num=100&hl=${country}`;
+      return `https://api.serply.io/v1/search/q=${encodeURIComponent(keyword.keyword)}&num=100&hl=${country}`;
    },
    resultObjectKey: 'result',
    serpExtractor: (content) => {

scrapers/services/spaceserp.ts

@@ -15,10 +15,10 @@ const spaceSerp:ScraperSettings = {
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
       const countryName = countries[country][0];
-      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
       const device = keyword.device === 'mobile' ? '&device=mobile' : '';
       const lang = countryData[country][2];
-      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
+      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

scrapers/services/valueserp.ts

@@ -15,11 +15,11 @@ const valueSerp:ScraperSettings = {
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
       const countryName = countries[country][0];
-      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
       const device = keyword.device === 'mobile' ? '&device=mobile' : '';
       const lang = countryData[country][2];
-      console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
+      console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
-      return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
+      return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

utils/adwords.ts

@@ -163,7 +163,7 @@ export const getAdwordsKeywordIdeas = async (credentials:AdwordsCredentials, adw
       }
 
       try {
-         // API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordIdeas
+         // API: https://developers.google.com/google-ads/api/rest/reference/rest/v18/customers/generateKeywordIdeas
          const customerID = account_id.replaceAll('-', '');
          const geoTargetConstants = countries[country][3]; // '2840';
          const reqPayload: Record<string, any> = {
@@ -178,7 +178,7 @@ export const getAdwordsKeywordIdeas = async (credentials:AdwordsCredentials, adw
             reqPayload.siteSeed = { site: domain };
          }
 
-         const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordIdeas`, {
+         const resp = await fetch(`https://googleads.googleapis.com/v18/customers/${customerID}:generateKeywordIdeas`, {
             method: 'POST',
             headers: {
                'Content-Type': 'application/json',
@@ -297,7 +297,7 @@ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error
       for (const country in keywordRequests) {
          if (Object.hasOwn(keywordRequests, country) && keywordRequests[country].length > 0) {
             try {
-               // API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordHistoricalMetrics
+               // API: https://developers.google.com/google-ads/api/rest/reference/rest/v18/customers/generateKeywordHistoricalMetrics
                const customerID = account_id.replaceAll('-', '');
                const geoTargetConstants = countries[country][3]; // '2840';
                const reqKeywords = keywordRequests[country].map((kw) => kw.keyword);
@@ -306,7 +306,7 @@ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error
                   geoTargetConstants: `geoTargetConstants/${geoTargetConstants}`,
                   // language: `languageConstants/${language}`,
                };
-               const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordHistoricalMetrics`, {
+               const resp = await fetch(`https://googleads.googleapis.com/v18/customers/${customerID}:generateKeywordHistoricalMetrics`, {
                   method: 'POST',
                   headers: {
                      'Content-Type': 'application/json',

utils/generateEmail.ts

@@ -4,10 +4,10 @@ import path from 'path';
 import { getKeywordsInsight, getPagesInsight } from './insight';
 import { readLocalSCData } from './searchConsole';
 
-const serpBearLogo = 'https://erevanto.sirv.com/Images/serpbear/ikAdjQq.png';
+const serpBearLogo = 'https://serpbear.b-cdn.net/ikAdjQq.png';
-const mobileIcon = 'https://erevanto.sirv.com/Images/serpbear/SqXD9rd.png';
+const mobileIcon = 'https://serpbear.b-cdn.net/SqXD9rd.png';
-const desktopIcon = 'https://erevanto.sirv.com/Images/serpbear/Dx3u0XD.png';
+const desktopIcon = 'https://serpbear.b-cdn.net/Dx3u0XD.png';
-const googleIcon = 'https://erevanto.sirv.com/Images/serpbear/Sx3u0X9.png';
+const googleIcon = 'https://serpbear.b-cdn.net/Sx3u0X9.png';
 
 type SCStatsObject = {
    [key:string]: {

utils/scraper.ts

@@ -1,5 +1,5 @@
 import axios, { AxiosResponse, CreateAxiosDefaults } from 'axios';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import { readFile, writeFile } from 'fs/promises';
 import HttpsProxyAgent from 'https-proxy-agent';
 import countries from './countries';
@@ -124,14 +124,18 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
          throw new Error(res);
       }
    } catch (error:any) {
-      refreshedResults.error = scraperError;
+      refreshedResults.error = scraperError || 'Unknown Error';
       if (settings.scraper_type === 'proxy' && error && error.response && error.response.statusText) {
          refreshedResults.error = `[${error.response.status}] ${error.response.statusText}`;
+      } else if (settings.scraper_type === 'proxy' && error) {
+         refreshedResults.error = error;
       }
 
-      console.log('[ERROR] Scraping Keyword : ', keyword.keyword, '. Error: ', error && error.response && error.response.statusText);
+      console.log('[ERROR] Scraping Keyword : ', keyword.keyword);
       if (!(error && error.response && error.response.statusText)) {
          console.log('[ERROR_MESSAGE]: ', error);
+      } else {
+         console.log('[ERROR_MESSAGE]: ', error && error.response && error.response.statusText);
       }
    }
 
@@ -148,9 +152,17 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
    const extractedResult = [];
 
    const $ = cheerio.load(content);
+   const hasValidContent = [...$('body').find('#search'), ...$('body').find('#rso')];
+   if (hasValidContent.length === 0) {
+      const msg = '[ERROR] Scraped search results do not adhere to expected format. Unable to parse results';
+      console.log(msg);
+      throw new Error(msg);
+   }
+
    const hasNumberofResult = $('body').find('#search  > div > div');
    const searchResultItems = hasNumberofResult.find('h3');
    let lastPosition = 0;
+   console.log('Scraped search results contain ', searchResultItems.length, ' desktop results.');
 
    for (let i = 0; i < searchResultItems.length; i += 1) {
       if (searchResultItems[i]) {
@@ -161,11 +173,12 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
             extractedResult.push({ title, url, position: lastPosition });
          }
       }
-  }
+   }
 
-  // Mobile Scraper
+   // Mobile Scraper
-  if (extractedResult.length === 0 && device === 'mobile') {
+   if (extractedResult.length === 0 && device === 'mobile') {
       const items = $('body').find('#rso > div');
+      console.log('Scraped search results contain ', items.length, ' mobile results.');
       for (let i = 0; i < items.length; i += 1) {
          const item = $(items[i]);
          const linkDom = item.find('a[role="presentation"]');
@@ -181,7 +194,7 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
       }
    }
 
-  return extractedResult;
+   return extractedResult;
 };
 
 /**