chore(release): 2.0.7

Add error message, if returned search HTML does not contain required elements

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [2.0.7](https://github.com/towfiqi/serpbear/compare/v2.0.6...v2.0.7) (2025-02-23)
+
+
+### Bug Fixes
+
+* Resolves AdWords integration issue. ([36ed4cf](https://github.com/towfiqi/serpbear/commit/36ed4cf800c1fd0e3df4e807faaa1fdb863df5e4))
+* resolves broken CDN images. ([bf911b4](https://github.com/towfiqi/serpbear/commit/bf911b4e45b9007a05ce6399838da3276161c61d))
+
+### [2.0.6](https://github.com/towfiqi/serpbear/compare/v2.0.5...v2.0.6) (2024-11-15)
+
+
+### Bug Fixes
+
+* Ensures Docker build uses matching npm package versions from package.json ([4fef1a9](https://github.com/towfiqi/serpbear/commit/4fef1a9abc737da67ab1ea0c4efce8194890545e))
+* Resolves broken Docker build due to croner package version mismatch. ([aeed1f8](https://github.com/towfiqi/serpbear/commit/aeed1f8559e044bf658d930a22fa91f38cfedc6b)), closes [#247](https://github.com/towfiqi/serpbear/issues/247)
+* Resolves broken Proxy Scraper functionality. ([649f412](https://github.com/towfiqi/serpbear/commit/649f412303dd50127b3736740962863f735f76eb)), closes [#248](https://github.com/towfiqi/serpbear/issues/248)
+* Resolves Google Ads search volume data loading issue. ([12eac2b](https://github.com/towfiqi/serpbear/commit/12eac2b01235e9eae06882d6a2c50c793b890661))
+
 ### [2.0.5](https://github.com/towfiqi/serpbear/compare/v2.0.4...v2.0.5) (2024-11-12)
 
 

Dockerfile

@@ -40,7 +40,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc
 COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv croner @googleapis/searchconsole sequelize-cli @isaacs/ttlcache
+RUN npm i cryptr@6.0.3 dotenv@16.0.3 croner@9.0.0 @googleapis/searchconsole@1.0.5 sequelize-cli@6.6.2 @isaacs/ttlcache@1.4.1
 RUN npm i -g concurrently
 
 USER nextjs

README.md

@@ -2,13 +2,13 @@
 
 # SerpBear
 
-![Codacy Badge](https://app.codacy.com/project/badge/Grade/7e7a0030c3f84c6fb56a3ce6273fbc1d) ![GitHub](https://img.shields.io/github/license/towfiqi/serpbear) ![GitHub package.json version](https://img.shields.io/github/package-json/v/towfiqi/serpbear) ![Docker Pulls](https://img.shields.io/docker/pulls/towfiqi/serpbear)
+![Codacy Badge](https://app.codacy.com/project/badge/Grade/7e7a0030c3f84c6fb56a3ce6273fbc1d) ![GitHub](https://img.shields.io/github/license/towfiqi/serpbear) ![GitHub package.json version](https://img.shields.io/github/package-json/v/towfiqi/serpbear) ![Docker Pulls](https://img.shields.io/docker/pulls/towfiqi/serpbear) [![StandWithPalestine](https://raw.githubusercontent.com/Safouene1/support-palestine-banner/master/StandWithPalestine.svg)](https://www.youtube.com/watch?v=bjtDsd0g468&rco=1)
 
 #### [Documentation](https://docs.serpbear.com/) | [Changelog](https://github.com/towfiqi/serpbear/blob/main/CHANGELOG.md) | [Docker Image](https://hub.docker.com/r/towfiqi/serpbear)
 
 SerpBear is an Open Source Search Engine Position Tracking and Keyword Research App. It allows you to track your website's keyword positions in Google and get notified of their position change.
 
-![Easy to Use Search Engine Rank Tracker](https://erevanto.sirv.com/Images/serpbear/serpbear_readme_v2.gif)
+![Easy to Use Search Engine Rank Tracker](https://serpbear.b-cdn.net/serpbear_readme_v2.gif)
 
 #### Features
 

__mocks__/utils.tsx

@@ -1,22 +1,26 @@
 import { render } from '@testing-library/react';
-import { rest } from 'msw';
+import { http } from 'msw';
 import * as React from 'react';
 import { QueryClient, QueryClientProvider } from 'react-query';
 
 export const handlers = [
-    rest.get(
+    http.get(
         '*/react-query',
-        (req, res, ctx) => {
-            return res(
-                ctx.status(200),
-                ctx.json({
+        ({ request, params }) => {
+            return new Response(
+                JSON.stringify({
                     name: 'mocked-react-query',
                 }),
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                },
             );
         },
     ),
 ];
-
 const createTestQueryClient = () => new QueryClient({
     defaultOptions: {
         queries: {

cron.js

@@ -1,7 +1,8 @@
+/* eslint-disable no-new */
 const Cryptr = require('cryptr');
 const { promises } = require('fs');
 const { readFile } = require('fs');
-const Cron = require('croner');
+const { Cron } = require('croner');
 require('dotenv').config({ path: './.env.local' });
 
 const getAppSettings = async () => {
@@ -71,7 +72,7 @@ const runAppCronJobs = () => {
       const scrape_interval = settings.scrape_interval || 'daily';
       if (scrape_interval !== 'never') {
          const scrapeCronTime = generateCronTime(scrape_interval);
-         Cron(scrapeCronTime, () => {
+         new Cron(scrapeCronTime, () => {
             // console.log('### Running Keyword Position Cron Job!');
             const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
             fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
@@ -89,7 +90,7 @@ const runAppCronJobs = () => {
       if (notif_interval) {
          const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
          if (cronTime) {
-            Cron(cronTime, () => {
+            new Cron(cronTime, () => {
                // console.log('### Sending Notification Email...');
                const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
                fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
@@ -106,7 +107,7 @@ const runAppCronJobs = () => {
 
    // Run Failed scraping CRON (Every Hour)
    const failedCronTime = generateCronTime('hourly');
-   Cron(failedCronTime, () => {
+   new Cron(failedCronTime, () => {
       // console.log('### Retrying Failed Scrapes...');
 
       readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
@@ -135,7 +136,7 @@ const runAppCronJobs = () => {
    // Run Google Search Console Scraper Daily
    if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
       const searchConsoleCRONTime = generateCronTime('daily');
-      Cron(searchConsoleCRONTime, () => {
+      new Cron(searchConsoleCRONTime, () => {
          const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
          fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts)
          .then((res) => res.json())

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "2.0.5",
+  "version": "2.0.7",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -18,16 +18,16 @@
     "release": "standard-version"
   },
   "dependencies": {
-    "@googleapis/searchconsole": "^1.0.0",
+    "@googleapis/searchconsole": "^1.0.5",
     "@isaacs/ttlcache": "^1.4.1",
     "@types/react-transition-group": "^4.4.5",
-    "axios": "^1.1.3",
+    "axios": "^1.7.7",
     "axios-retry": "^3.3.1",
     "chart.js": "^3.9.1",
     "cheerio": "^1.0.0",
     "concurrently": "^7.6.0",
     "cookies": "^0.8.0",
-    "croner": "^5.3.5",
+    "croner": "^9.0.0",
     "cryptr": "^6.0.3",
     "dayjs": "^1.11.5",
     "dotenv": "^16.0.3",
@@ -35,7 +35,6 @@
     "https-proxy-agent": "^5.0.1",
     "isomorphic-fetch": "^3.0.0",
     "jsonwebtoken": "^9.0.2",
-    "msw": "^0.49.0",
     "next": "^12.3.4",
     "nodemailer": "^6.9.9",
     "react": "18.2.0",
@@ -50,11 +49,11 @@
     "reflect-metadata": "^0.1.13",
     "sequelize": "^6.34.0",
     "sequelize-typescript": "^2.1.6",
-    "sqlite3": "^5.1.6",
-    "umzug": "^3.6.1"
+    "sqlite3": "^5.1.7",
+    "umzug": "^3.8.2"
   },
   "devDependencies": {
-    "@testing-library/jest-dom": "^6.1.4",
+    "@testing-library/jest-dom": "^6.6.3",
     "@testing-library/react": "^14.0.0",
     "@types/cookies": "^0.7.7",
     "@types/cryptr": "^4.0.1",
@@ -74,15 +73,16 @@
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "jest-fetch-mock": "^3.0.3",
+    "msw": "^2.6.4",
     "next-router-mock": "^0.9.10",
-    "postcss": "^8.4.31",
+    "postcss": "^8.4.49",
     "prettier": "^2.7.1",
     "resize-observer-polyfill": "^1.5.1",
-    "sass": "^1.55.0",
+    "sass": "^1.80.7",
     "sequelize-cli": "^6.6.2",
     "standard-version": "^9.5.0",
     "stylelint-config-standard": "^29.0.0",
-    "tailwindcss": "^3.1.8",
-    "typescript": "4.8.4"
+    "tailwindcss": "^3.4.14",
+    "typescript": "^4.8.4"
   }
 }

pages/api/volume.ts

@@ -36,14 +36,13 @@ const updatekeywordVolume = async (req: NextApiRequest, res: NextApiResponse<Key
          keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
       }
       if (domain) {
-         // const allDomain = domain === 'all';
-         // const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
-         // keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
+         const allDomain = domain === 'all';
+         const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
+         keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
       }
 
       if (keywordsToSend.length > 0) {
          const keywordsVolumeData = await getKeywordsVolume(keywordsToSend);
-         // console.log('keywordsVolumeData :', keywordsVolumeData);
          if (keywordsVolumeData.error) {
             return res.status(400).json({ keywords: [], error: keywordsVolumeData.error });
          }

scrapers/services/proxy.ts

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 const proxy:ScraperSettings = {
    id: 'proxy',
@@ -16,6 +16,13 @@ const proxy:ScraperSettings = {
 
       const $ = cheerio.load(content);
       let lastPosition = 0;
+      const hasValidContent = $('body').find('#main');
+      if (hasValidContent.length === 0) {
+         const msg = '[ERROR] Scraped search results from proxy do not adhere to expected format. Unable to parse results';
+         console.log(msg);
+         throw new Error(msg);
+      }
+
       const mainContent = $('body').find('#main');
       const children = $(mainContent).find('h3');
 

utils/adwords.ts

@@ -163,7 +163,7 @@ export const getAdwordsKeywordIdeas = async (credentials:AdwordsCredentials, adw
       }
 
       try {
-         // API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordIdeas
+         // API: https://developers.google.com/google-ads/api/rest/reference/rest/v18/customers/generateKeywordIdeas
          const customerID = account_id.replaceAll('-', '');
          const geoTargetConstants = countries[country][3]; // '2840';
          const reqPayload: Record<string, any> = {
@@ -178,7 +178,7 @@ export const getAdwordsKeywordIdeas = async (credentials:AdwordsCredentials, adw
             reqPayload.siteSeed = { site: domain };
          }
 
-         const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordIdeas`, {
+         const resp = await fetch(`https://googleads.googleapis.com/v18/customers/${customerID}:generateKeywordIdeas`, {
             method: 'POST',
             headers: {
                'Content-Type': 'application/json',
@@ -297,7 +297,7 @@ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error
       for (const country in keywordRequests) {
          if (Object.hasOwn(keywordRequests, country) && keywordRequests[country].length > 0) {
             try {
-               // API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordHistoricalMetrics
+               // API: https://developers.google.com/google-ads/api/rest/reference/rest/v18/customers/generateKeywordHistoricalMetrics
                const customerID = account_id.replaceAll('-', '');
                const geoTargetConstants = countries[country][3]; // '2840';
                const reqKeywords = keywordRequests[country].map((kw) => kw.keyword);
@@ -306,7 +306,7 @@ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error
                   geoTargetConstants: `geoTargetConstants/${geoTargetConstants}`,
                   // language: `languageConstants/${language}`,
                };
-               const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordHistoricalMetrics`, {
+               const resp = await fetch(`https://googleads.googleapis.com/v18/customers/${customerID}:generateKeywordHistoricalMetrics`, {
                   method: 'POST',
                   headers: {
                      'Content-Type': 'application/json',

utils/generateEmail.ts

@@ -4,10 +4,10 @@ import path from 'path';
 import { getKeywordsInsight, getPagesInsight } from './insight';
 import { readLocalSCData } from './searchConsole';
 
-const serpBearLogo = 'https://erevanto.sirv.com/Images/serpbear/ikAdjQq.png';
-const mobileIcon = 'https://erevanto.sirv.com/Images/serpbear/SqXD9rd.png';
-const desktopIcon = 'https://erevanto.sirv.com/Images/serpbear/Dx3u0XD.png';
-const googleIcon = 'https://erevanto.sirv.com/Images/serpbear/Sx3u0X9.png';
+const serpBearLogo = 'https://serpbear.b-cdn.net/ikAdjQq.png';
+const mobileIcon = 'https://serpbear.b-cdn.net/SqXD9rd.png';
+const desktopIcon = 'https://serpbear.b-cdn.net/Dx3u0XD.png';
+const googleIcon = 'https://serpbear.b-cdn.net/Sx3u0X9.png';
 
 type SCStatsObject = {
    [key:string]: {

utils/scraper.ts

@@ -127,11 +127,15 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
       refreshedResults.error = scraperError || 'Unknown Error';
       if (settings.scraper_type === 'proxy' && error && error.response && error.response.statusText) {
          refreshedResults.error = `[${error.response.status}] ${error.response.statusText}`;
+      } else if (settings.scraper_type === 'proxy' && error) {
+         refreshedResults.error = error;
       }
 
-      console.log('[ERROR] Scraping Keyword : ', keyword.keyword, '. Error: ', error && error.response && error.response.statusText);
+      console.log('[ERROR] Scraping Keyword : ', keyword.keyword);
       if (!(error && error.response && error.response.statusText)) {
          console.log('[ERROR_MESSAGE]: ', error);
+      } else {
+         console.log('[ERROR_MESSAGE]: ', error && error.response && error.response.statusText);
       }
    }
 
@@ -148,9 +152,17 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
    const extractedResult = [];
 
    const $ = cheerio.load(content);
+   const hasValidContent = [...$('body').find('#search'), ...$('body').find('#rso')];
+   if (hasValidContent.length === 0) {
+      const msg = '[ERROR] Scraped search results do not adhere to expected format. Unable to parse results';
+      console.log(msg);
+      throw new Error(msg);
+   }
+
    const hasNumberofResult = $('body').find('#search  > div > div');
    const searchResultItems = hasNumberofResult.find('h3');
    let lastPosition = 0;
+   console.log('Scraped search results contain ', searchResultItems.length, ' desktop results.');
 
    for (let i = 0; i < searchResultItems.length; i += 1) {
       if (searchResultItems[i]) {
@@ -161,11 +173,12 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
             extractedResult.push({ title, url, position: lastPosition });
          }
       }
-  }
+   }
 
-  // Mobile Scraper
-  if (extractedResult.length === 0 && device === 'mobile') {
+   // Mobile Scraper
+   if (extractedResult.length === 0 && device === 'mobile') {
       const items = $('body').find('#rso > div');
+      console.log('Scraped search results contain ', items.length, ' mobile results.');
       for (let i = 0; i < items.length; i += 1) {
          const item = $(items[i]);
          const linkDom = item.find('a[role="presentation"]');
@@ -181,7 +194,7 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
       }
    }
 
-  return extractedResult;
+   return extractedResult;
 };
 
 /**