chore(release): 2.0.3

closes 202

.eslintrc.json

@@ -14,6 +14,8 @@
    "max-len": ["error", {"code": 150, "ignoreComments": true, "ignoreUrls": true}],
    "import/no-extraneous-dependencies": "off",
    "no-unused-vars": "off",
+   "implicit-arrow-linebreak": "off",
+   "function-paren-newline": "off",
    "import/extensions": [
       "error",
       "ignorePackages",

CHANGELOG.md

@@ -2,6 +2,46 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [2.0.3](https://github.com/towfiqi/serpbear/compare/v2.0.2...v2.0.3) (2024-11-10)
+
+
+### Features
+
+* Ability to keywords for both mobile and desktop at once. ([3786438](https://github.com/towfiqi/serpbear/commit/3786438662f015613330912f668c161f007f3eef)), closes [#60](https://github.com/towfiqi/serpbear/issues/60) [#66](https://github.com/towfiqi/serpbear/issues/66) [#199](https://github.com/towfiqi/serpbear/issues/199)
+* Adds ability to set Notification Email From name. ([b35d333](https://github.com/towfiqi/serpbear/commit/b35d333bfcdf0765bd105f74c8b324aec6d98b22)), closes [#222](https://github.com/towfiqi/serpbear/issues/222)
+* Adds the ability to show hide columns in tracked keywords table. ([d3e3760](https://github.com/towfiqi/serpbear/commit/d3e37605279c48f7a56b2ef5125c32a00b948e05)), closes [#224](https://github.com/towfiqi/serpbear/issues/224)
+* auto filter keywords if they already exist instead of throwing error. ([a09eb62](https://github.com/towfiqi/serpbear/commit/a09eb62f5a65f584c32ddb5c46018dc404d3e1f3)), closes [#244](https://github.com/towfiqi/serpbear/issues/244)
+* Displays Best position on mobile layout as well. ([a74338f](https://github.com/towfiqi/serpbear/commit/a74338fe15bee0fbc65a5197ee56db5d6a629bc8))
+* Displays keyword's best position in email notification. ([4c2f900](https://github.com/towfiqi/serpbear/commit/4c2f900d85e56b1994630942c9ea53d0fd0b4cdb))
+* Keywords Country filter now only shows relevant countries. ([a050536](https://github.com/towfiqi/serpbear/commit/a0505368140076d6626647993a41a5f4ef9db019))
+* Makes Content width a little wider. ([42c5e2b](https://github.com/towfiqi/serpbear/commit/42c5e2be0777a1bd72aa53bdf3c211075793a6e8))
+
+
+### Bug Fixes
+
+* Correct CTR calculation in InsightStats component ([232507e](https://github.com/towfiqi/serpbear/commit/232507e1ff7519cfc82f3818d3b757f2c427b52a))
+* Fixes incorrect position display in keyword detail view when position is above 100 ([01b1b7b](https://github.com/towfiqi/serpbear/commit/01b1b7b9e9c603e470217ad476dd54359d7d35b8))
+* Fixes missing keyword city value in exported csv file. ([f482884](https://github.com/towfiqi/serpbear/commit/f48288473e910c00e7c9178366a394e891bc47c7)), closes [#194](https://github.com/towfiqi/serpbear/issues/194)
+* Fixes missing Search Console data in Email notification when its integrated through App settings. ([040dab1](https://github.com/towfiqi/serpbear/commit/040dab15177d81874a6eb89f913a450f8a6f212d))
+* Resolves incorrect search trend graph in Ideas section. ([7597210](https://github.com/towfiqi/serpbear/commit/7597210ca2018b4abc82d441d2fc871e46295307)), closes [#219](https://github.com/towfiqi/serpbear/issues/219)
+* Resolves notification email's incorrect image size in some email clients. ([bc96dc7](https://github.com/towfiqi/serpbear/commit/bc96dc7de50844fbda19a89961c36cc65ac0b97b)), closes [#201](https://github.com/towfiqi/serpbear/issues/201)
+* update scraping robot typo in README ([c24b630](https://github.com/towfiqi/serpbear/commit/c24b63009c48c61218ea96511b11fe1a4f2ff239))
+
+### [2.0.2](https://github.com/towfiqi/serpbear/compare/v2.0.1...v2.0.2) (2024-03-13)
+
+
+### Bug Fixes
+
+* Resolves Broken Google Adwords Authentication in Docker containers. ([1d0a788](https://github.com/towfiqi/serpbear/commit/1d0a78881039b3ebd4b1e00fb067016886d49c31)), closes [#179](https://github.com/towfiqi/serpbear/issues/179)
+
+### [2.0.1](https://github.com/towfiqi/serpbear/compare/v2.0.0...v2.0.1) (2024-03-06)
+
+
+### Bug Fixes
+
+* Resolves broken doc links ([d48ae76](https://github.com/towfiqi/serpbear/commit/d48ae76103615ab82a32cbd8ffa62e27d17999de))
+* Resolves keyword loading issue in Docker instances. ([4a87d22](https://github.com/towfiqi/serpbear/commit/4a87d229fee7aa5e0ca8b2042c168465c7c5d67f)), closes [#178](https://github.com/towfiqi/serpbear/issues/178)
+
 ## [2.0.0](https://github.com/towfiqi/serpbear/compare/v1.0.3...v2.0.0) (2024-03-05)
 
 

Dockerfile

@@ -35,7 +35,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc
 COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv croner @googleapis/searchconsole sequelize-cli
+RUN npm i cryptr dotenv croner @googleapis/searchconsole sequelize-cli @isaacs/ttlcache
 RUN npm i -g concurrently
 
 USER nextjs

README.md

@@ -22,11 +22,11 @@ SerpBear is an Open Source Search Engine Position Tracking and Keyword Research
 
 #### How it Works
 
-The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword.
+The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi, HasData or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword.
 
-The Keyword Research and keyword generation feature works by integrating your Google Ads test accounts into SerpBear. You can also view the added keyword's monthly search volume data once you [integrate Google Ads](/miscellaneous/integrate-google-ads).
+The Keyword Research and keyword generation feature works by integrating your Google Ads test accounts into SerpBear. You can also view the added keyword's monthly search volume data once you [integrate Google Ads](https://docs.serpbear.com/miscellaneous/integrate-google-ads).
 
-When you [integrate Google Search Console](/miscellaneous/integrate-google-search-console), the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages.you will be able to view the actual visits count from Google Search for the tracked keywords.
+When you [integrate Google Search Console](https://docs.serpbear.com/miscellaneous/integrate-google-search-console), the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages.you will be able to view the actual visits count from Google Search for the tracked keywords.
 
 #### Getting Started
 
@@ -43,15 +43,16 @@ When you [integrate Google Search Console](/miscellaneous/integrate-google-searc
 If you don't want to use proxies, you can use third party Scraping services to scrape Google Search results.
 
 
-| Service          | Cost          | SERP Lookup    | API |
-| ---------------- | ------------- | -------------- | --- |
-| scraingrobot.com | Free          | 5000/mo        | Yes |
-| serply.io        | $49/mo        | 5000/mo        | Yes |
-| serpapi.com      | From $50/mo   | From 5,000/mo  | Yes |
-| spaceserp.com    | $59/lifetime  | 15,000/mo      | Yes |
-| SearchApi.io     | From $40/mo   | From 10,000/mo | Yes |
-| valueserp.com    | Pay As You Go | $2.50/1000 req | No  |
-| serper.dev       | Pay As You Go | $1.00/1000 req | No  |
+| Service           | Cost          | SERP Lookup    | API |
+| ----------------- | ------------- | -------------- | --- |
+| scrapingrobot.com | Free          | 5000/mo        | Yes |
+| serply.io         | $49/mo        | 5000/mo        | Yes |
+| serpapi.com       | From $50/mo   | From 5,000/mo  | Yes |
+| spaceserp.com     | $59/lifetime  | 15,000/mo      | Yes |
+| SearchApi.io      | From $40/mo   | From 10,000/mo | Yes |
+| valueserp.com     | Pay As You Go | $2.50/1000 req | No  |
+| serper.dev        | Pay As You Go | $1.00/1000 req | No  |
+| hasdata.com       | From $29/mo   | From 10,000/mo | Yes |
 
 **Tech Stack**
 

__mocks__/data.ts

@@ -69,6 +69,7 @@ export const dummySettings = {
    notification_interval: 'never',
    notification_email: '',
    notification_email_from: '',
+   notification_email_from_name: 'SerpBear',
    smtp_server: '',
    smtp_port: '',
    smtp_username: '',

components/common/ChartSlim.tsx

@@ -7,10 +7,11 @@ ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Filler,
 type ChartProps ={
    labels: string[],
    sreies: number[],
-   noMaxLimit?: boolean
+   noMaxLimit?: boolean,
+   reverse?: boolean
 }
 
-const ChartSlim = ({ labels, sreies, noMaxLimit = false }:ChartProps) => {
+const ChartSlim = ({ labels, sreies, noMaxLimit = false, reverse = true }:ChartProps) => {
    const options = {
       responsive: true,
       maintainAspectRatio: false,
@@ -18,7 +19,7 @@ const ChartSlim = ({ labels, sreies, noMaxLimit = false }:ChartProps) => {
       scales: {
          y: {
             display: false,
-            reverse: true,
+            reverse,
             min: 1,
             max: noMaxLimit ? undefined : 100,
          },

components/common/Icon.tsx

@@ -10,10 +10,10 @@ type IconProps = {
 }
 
 const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '' }: IconProps) => {
-   const xmlnsProps = { xmlns: 'http://www.w3.org/2000/svg', xmlnsXlink: 'http://www.w3.org/1999/xlink', preserveAspectRatio: 'xMidYMid meet' };
+   const xmlnsProps = { title, xmlns: 'http://www.w3.org/2000/svg', xmlnsXlink: 'http://www.w3.org/1999/xlink', preserveAspectRatio: 'xMidYMid meet' };
 
    return (
-       <span className={`icon inline-block relative top-[2px] ${classes}`} title={title}>
+       <span className={`icon inline-block relative top-[2px] ${classes}`}>
          {type === 'logo'
             && <svg {...xmlnsProps} width={size} viewBox="0 0 1484.32 1348.5">
                <path fill={color} d="M1406.23,604.17s-44-158.18,40.43-192.67,195,97.52,195,97.52,314-65.41,534,0c0,0,122.16-105.61,214.68-80.28,99.9,27.36,32.7,181.38,32.7,181.38s228.36,384.15,239.06,737.38c0,0-346.1,346.09-746.9,406.75,0,0-527.47-106.44-737.38-449.57C1177.88,1304.68,1169.55,1008.54,1406.23,604.17Z" transform="translate(-1177.84 -405.75)"/>
@@ -308,6 +308,11 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
                   <path fill={color} d="M7.328 43.504c.445 0 .914-.14 1.383-.469V17.957c0-.844.164-1.172.914-1.57L31.352 3.87c.07-1.547-.915-2.508-2.25-2.508c-.61 0-1.266.164-1.97.586L7.493 13.246c-2.297 1.336-2.578 1.758-2.578 4.43v22.43c0 2.015.96 3.398 2.414 3.398m9.375 5.414c.422 0 .89-.14 1.383-.469V23.371c0-.914.117-1.148.89-1.57L40.703 9.26c.07-1.523-.89-2.507-2.25-2.507c-.586 0-1.266.187-1.945.562L16.82 18.636c-2.297 1.313-2.555 1.805-2.555 4.43V45.52c0 2.015 1.008 3.398 2.438 3.398m10.031 5.719c.82 0 1.805-.328 2.977-.985l18.375-10.547c2.156-1.242 3-2.53 3-5.156l-.047-21.234c0-2.813-1.008-4.242-2.766-4.242c-.773 0-1.758.304-2.859.937L26.992 24.027c-2.203 1.29-2.977 2.602-2.977 5.157v21.234c0 2.719.961 4.219 2.72 4.219M28 50.067c-.117-.024-.164-.094-.164-.258L28 29.254c0-.89.258-1.36 1.055-1.805l17.742-10.43c.07-.046.14-.046.234-.023c.094.024.164.094.164.258l-.07 20.625c0 .773-.281 1.36-1.055 1.828L28.234 50.043a.284.284 0 0 1-.234.023"></path>
                </svg>
          }
+         {type === 'lock'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill={color} d="M6 22q-.825 0-1.412-.587T4 20V10q0-.825.588-1.412T6 8h1V6q0-2.075 1.463-3.537T12 1t3.538 1.463T17 6v2h1q.825 0 1.413.588T20 10v10q0 .825-.587 1.413T18 22zm6-5q.825 0 1.413-.587T14 15t-.587-1.412T12 13t-1.412.588T10 15t.588 1.413T12 17M9 8h6V6q0-1.25-.875-2.125T12 3t-2.125.875T9 6z" />
+               </svg>
+         }
        </span>
    );
  };

components/ideas/KeywordIdea.tsx

@@ -61,7 +61,7 @@ const KeywordIdea = (props: KeywordIdeaProps) => {
           onClick={() => showKeywordDetails()}
          className={`keyword_visits text-center hidden mt-4 mr-5 ml-5 cursor-pointer
          lg:flex-1 lg:m-0 lg:ml-10 max-w-[70px] lg:max-w-none lg:pr-5 lg:flex justify-center`}>
-            {chartData.labels.length > 0 && <ChartSlim labels={chartData.labels} sreies={chartData.sreies} noMaxLimit={true} />}
+            {chartData.labels.length > 0 && <ChartSlim labels={chartData.labels} sreies={chartData.sreies} noMaxLimit={true} reverse={false} />}
          </div>
 
          <div className='keyword_ctr text-center inline-block ml-4 lg:flex mt-4 relative lg:flex-1 lg:m-0 justify-center'>

components/insight/InsightStats.tsx

@@ -13,16 +13,20 @@ type InsightStatsProps = {
 }
 
 const InsightStats = ({ stats = [], totalKeywords = 0, totalPages = 0 }:InsightStatsProps) => {
-   const totalStat = useMemo(() => {
-      return stats.reduce((acc, item) => {
-         return {
+    const totalStat = useMemo(() => {
+      const totals = stats.reduce((acc, item) => {
+          return {
             impressions: item.impressions + acc.impressions,
             clicks: item.clicks + acc.clicks,
-            ctr: item.ctr + acc.ctr,
             position: item.position + acc.position,
-         };
-      }, { impressions: 0, clicks: 0, ctr: 0, position: 0 });
-   }, [stats]);
+          };
+      }, { impressions: 0, clicks: 0, position: 0 });
+
+      return {
+          ...totals,
+          ctr: totals.impressions > 0 ? (totals.clicks / totals.impressions) * 100 : 0,
+      };
+    }, [stats]);
 
    const chartData = useMemo(() => {
       const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];

components/keywords/AddKeywords.tsx

@@ -1,4 +1,4 @@
-import React, { useMemo, useRef, useState } from 'react';
+import React, { useCallback, useMemo, useRef, useState } from 'react';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
 import SelectField from '../common/SelectField';
@@ -23,10 +23,11 @@ type KeywordsInput = {
 }
 
 const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCity = false }: AddKeywordsProps) => {
-   const [error, setError] = useState<string>('');
-   const [showTagSuggestions, setShowTagSuggestions] = useState(false);
    const inputRef = useRef(null);
    const defCountry = localStorage.getItem('default_country') || 'US';
+
+   const [error, setError] = useState<string>('');
+   const [showTagSuggestions, setShowTagSuggestions] = useState(false);
    const [newKeywordsData, setNewKeywordsData] = useState<KeywordsInput>({ keywords: '', device: 'desktop', country: defCountry, domain, tags: '' });
    const { mutate: addMutate, isLoading: isAdding } = useAddKeywords(() => closeModal(false));
 
@@ -35,20 +36,45 @@ const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCit
       return [...new Set(allTags)];
    }, [keywords]);
 
+   const setDeviceType = useCallback((input:string) => {
+      let updatedDevice = '';
+      if (newKeywordsData.device.includes(input)) {
+         updatedDevice = newKeywordsData.device.replace(',', '').replace(input, '');
+      } else {
+         updatedDevice = newKeywordsData.device ? `${newKeywordsData.device},${input}` : input;
+      }
+      setNewKeywordsData({ ...newKeywordsData, device: updatedDevice });
+   }, [newKeywordsData]);
+
    const addKeywords = () => {
-      if (newKeywordsData.keywords) {
-         const keywordsArray = [...new Set(newKeywordsData.keywords.split('\n').map((item) => item.trim()).filter((item) => !!item))];
+      const nkwrds = newKeywordsData;
+      if (nkwrds.keywords) {
+         const devices = nkwrds.device.split(',');
+         const multiDevice = nkwrds.device.includes(',') && devices.length > 1;
+         const keywordsArray = [...new Set(nkwrds.keywords.split('\n').map((item) => item.trim()).filter((item) => !!item))];
          const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}${k.city ? `-${k.city}` : ''}`);
-         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(
-            `${k}-${newKeywordsData.device}-${newKeywordsData.country}${newKeywordsData.city ? `-${newKeywordsData.city}` : ''}`,
-         ));
-         if (keywordExist.length > 0) {
+
+         const keywordExist = keywordsArray.filter((k) =>
+            devices.some((device) => currentKeywords.includes(`${k}-${device}-${nkwrds.country}${nkwrds.city ? `-${nkwrds.city}` : ''}`)),
+         );
+
+         if (!multiDevice && (keywordsArray.length === 1 || currentKeywords.length === keywordExist.length) && keywordExist.length > 0) {
             setError(`Keywords ${keywordExist.join(',')} already Exist`);
             setTimeout(() => { setError(''); }, 3000);
          } else {
-            const { device, country, domain: kDomain, tags, city } = newKeywordsData;
-            const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags, city }));
-            addMutate(newKeywordsArray);
+            const newKeywords = keywordsArray.flatMap((k) =>
+               devices.filter((device) =>
+                 !currentKeywords.includes(`${k}-${device}-${nkwrds.country}${nkwrds.city ? `-${nkwrds.city}` : ''}`),
+               ).map((device) => ({
+                 keyword: k,
+                 device,
+                 country: nkwrds.country,
+                 domain: nkwrds.domain,
+                 tags: nkwrds.tags,
+                 city: nkwrds.city,
+               })),
+             );
+            addMutate(newKeywords);
          }
       } else {
          setError('Please Insert a Keyword');
@@ -56,7 +82,7 @@ const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCit
       }
    };
 
-   const deviceTabStyle = 'cursor-pointer px-3 py-2 rounded mr-2';
+   const deviceTabStyle = 'cursor-pointer px-2 py-2 rounded';
 
    return (
       <Modal closeModal={() => { closeModal(false); }} title={'Add New Keywords'} width="[420px]">
@@ -89,13 +115,17 @@ const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCit
                   </div>
                   <ul className='flex text-xs font-semibold text-gray-500'>
                      <li
-                        className={`${deviceTabStyle} ${newKeywordsData.device === 'desktop' ? '  bg-indigo-50 text-gray-700' : ''}`}
-                        onClick={() => setNewKeywordsData({ ...newKeywordsData, device: 'desktop' })}
-                        ><Icon type='desktop' classes={'top-[3px]'} size={15} /> <i className='not-italic hidden lg:inline-block'>Desktop</i></li>
+                        className={`${deviceTabStyle} mr-2 ${newKeywordsData.device.includes('desktop') ? '  bg-indigo-50 text-indigo-700' : ''}`}
+                        onClick={() => setDeviceType('desktop')}>
+                           <Icon type='desktop' classes={'top-[3px]'} size={15} /> <i className='not-italic hidden lg:inline-block'>Desktop</i>
+                           <Icon type='check' classes={'pl-1'} size={12} color={newKeywordsData.device.includes('desktop') ? '#4338ca' : '#bbb'} />
+                        </li>
                      <li
-                        className={`${deviceTabStyle} ${newKeywordsData.device === 'mobile' ? '  bg-indigo-50 text-gray-700' : ''}`}
-                        onClick={() => setNewKeywordsData({ ...newKeywordsData, device: 'mobile' })}
-                        ><Icon type='mobile' /> <i className='not-italic hidden lg:inline-block'>Mobile</i></li>
+                        className={`${deviceTabStyle} ${newKeywordsData.device.includes('mobile') ? '  bg-indigo-50 text-indigo-700' : ''}`}
+                        onClick={() => setDeviceType('mobile')}>
+                           <Icon type='mobile' /> <i className='not-italic hidden lg:inline-block'>Mobile</i>
+                           <Icon type='check' classes={'pl-1'} size={12} color={newKeywordsData.device.includes('mobile') ? '#4338ca' : '#bbb'} />
+                        </li>
                   </ul>
                </div>
                <div className='relative'>

components/keywords/Keyword.tsx

@@ -21,7 +21,9 @@ type KeywordProps = {
    lastItem?:boolean,
    showSCData: boolean,
    scDataType: string,
-   style: Object
+   style: Object,
+   maxTitleColumnWidth: number,
+   tableColumns? : string[]
 }
 
 const Keyword = (props: KeywordProps) => {
@@ -39,12 +41,16 @@ const Keyword = (props: KeywordProps) => {
       style,
       index,
       scDataType = 'threeDays',
+      tableColumns = [],
+      maxTitleColumnWidth,
    } = props;
    const {
       keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false, volume,
    } = keywordData;
+
    const [showOptions, setShowOptions] = useState(false);
    const [showPositionError, setPositionError] = useState(false);
+
    const turncatedURL = useMemo(() => {
       return url.replace(`https://${domain}`, '').replace(`https://www.${domain}`, '').replace(`http://${domain}`, '');
    }, [url, domain]);
@@ -73,7 +79,7 @@ const Keyword = (props: KeywordProps) => {
       let bestPos;
       if (Object.keys(history).length > 0) {
          const historyArray = Object.keys(history).map((itemID) => ({ date: itemID, position: history[itemID] }))
-         .sort((a, b) => a.position - b.position);
+             .sort((a, b) => a.position - b.position).filter((el) => (el.position > 0));
          if (historyArray[0]) {
             bestPos = { ...historyArray[0] };
          }
@@ -91,7 +97,7 @@ const Keyword = (props: KeywordProps) => {
       className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
       lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
 
-         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
+         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
             <button
                className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
                ${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
@@ -100,10 +106,15 @@ const Keyword = (props: KeywordProps) => {
                   <Icon type="check" size={10} />
             </button>
             <a
-            className={`py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full ${showSCData ? 'lg:max-w-[180px]' : 'lg:max-w-[240px]'}`}
-            onClick={() => showKeywordDetails()}>
+            style={{ maxWidth: `${maxTitleColumnWidth - 35}px` }}
+            className={'py-2 hover:text-blue-600 lg:flex lg:items-center w-full'}
+            onClick={() => showKeywordDetails()}
+            title={keyword}
+            >
                <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />
-               <span className=' text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-30px)]'>{keyword}{city ? ` (${city})` : ''}</span>
+               <span className='inline-block text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-50px)]'>
+                  {keyword}{city ? ` (${city})` : ''}
+               </span>
             </a>
             {sticky && <button className='ml-2 relative top-[2px]' title='Favorite'><Icon type="star-filled" size={16} color="#fbd346" /></button>}
             {lastUpdateError && lastUpdateError.date
@@ -125,41 +136,45 @@ const Keyword = (props: KeywordProps) => {
          title={bestPosition && bestPosition.date
             ? new Date(bestPosition.date).toLocaleDateString('en-US', { weekday: 'long', year: 'numeric', month: 'short', day: 'numeric' }) : ''
          }
-         className={`keyword_best hidden bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative lg:block
-          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-16 lg:grow-0 lg:right-0 text-center font-semibold`}>
+         className={`keyword_best mr-1 bg-[#f8f9ff] float-right mt-8 w-14 rounded right-5 lg:relative lg:block
+          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:mr-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-16 lg:grow-0 lg:right-0 text-center font-semibold
+          ${!tableColumns.includes('Best') ? 'lg:hidden' : ''}
+          `}>
             {bestPosition ? bestPosition.position || '-' : (position || '-')}
          </div>
 
          {chartData.labels.length > 0 && (
             <div
-               className='hidden basis-20 grow-0 cursor-pointer lg:block'
+               className={`hidden basis-20 grow-0 cursor-pointer lg:block ${!tableColumns.includes('History') ? 'lg:hidden' : ''}`}
                onClick={() => showKeywordDetails()}>
                <ChartSlim labels={chartData.labels} sreies={chartData.sreies} />
             </div>
          )}
 
          <div
-         className={`keyword_best hidden bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative lg:block
-          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center`}>
+         className={`hidden bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative lg:block
+          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center
+          ${!tableColumns.includes('Volume') ? 'lg:hidden' : ''}
+          `}>
             {formattedNum(volume)}
          </div>
 
          <div
          className={`keyword_url inline-block mt-4 mr-5 ml-5 lg:flex-1 text-gray-400 lg:m-0 max-w-[70px] 
-         overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5`}>
+         overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5 lg:pl-3`}>
             <a href={url} target="_blank" rel="noreferrer"><span className='mr-3 lg:hidden'>
                <Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}
             </a>
          </div>
 
          <div
-         className='inline-block mt-[4] top-[-5px] relative lg:flex-1 lg:m-0 lg:top-0'>
+         className='inline-block mt-[4] top-[-5px] relative lg:flex-1 lg:m-0 lg:top-0 max-w-[150px]'>
             <span className='mr-2 lg:hidden'><Icon type="clock" size={14} color="#999" /></span>
             <TimeAgo title={dayjs(lastUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={lastUpdated} />
          </div>
 
-         {showSCData && (
-            <div className='keyword_sc_data min-w-[170px] text-xs mt-4 pt-2 border-t border-gray-100 top-[6px]
+         {showSCData && tableColumns.includes('Search Console') && (
+            <div className='keyword_sc_data min-w-[170px] lg:max-w-[170px] text-xs mt-4 pt-2 border-t border-gray-100 top-[6px]
             relative flex justify-between text-center lg:flex-1 lg:text-sm lg:m-0 lg:mt-0 lg:border-t-0 lg:pt-0 lg:top-0'>
                <span className='min-w-[40px]'>
                   <span className='lg:hidden'>SC Position: </span>

components/keywords/KeywordDetails.tsx

@@ -54,11 +54,14 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
    return (
        <div className="keywordDetails fixed w-full h-screen top-0 left-0 z-[99999]" onClick={closeOnBGClick} data-testid="keywordDetails">
             <div className="keywordDetails absolute w-full lg:w-5/12 bg-white customShadow top-0 right-0 h-screen" >
-               <div className='keywordDetails__header p-6 border-b border-b-slate-200 text-slate-500'>
+               <div className='keywordDetails__header p-6 border-b border-b-slate-200 text-slate-600'>
                   <h3 className=' text-lg font-bold'>
                      <span title={countries[keyword.country][0]}
                      className={`fflag fflag-${keyword.country} w-[18px] h-[12px] mr-2`} /> {keyword.keyword}
-                     <span className='py-1 px-2 ml-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
+                     <span
+                     className={`py-1 px-2 ml-2 rounded bg-blue-50 ${keyword.position === 0 ? 'text-gray-500' : 'text-blue-700'}  text-xs font-bold`}>
+                        {keyword.position === 0 ? 'Not in First 100' : keyword.position}
+                     </span>
                   </h3>
                   <button
                   className='absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'

components/keywords/KeywordFilter.tsx

@@ -15,6 +15,8 @@ type KeywordFilterProps = {
    integratedConsole?: boolean,
    isConsole?: boolean,
    SCcountries?: string[];
+   updateColumns?: Function,
+   tableColumns?: string[]
 }
 
 const KeywordFilters = (props: KeywordFilterProps) => {
@@ -29,10 +31,13 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       filterParams,
       isConsole = false,
       integratedConsole = false,
+      updateColumns,
       SCcountries = [],
+      tableColumns = [],
     } = props;
    const [sortOptions, showSortOptions] = useState(false);
    const [filterOptions, showFilterOptions] = useState(false);
+   const [columnOptions, showColumnOptions] = useState(false);
 
    const keywordCounts = useMemo(() => {
       const counts = { desktop: 0, mobile: 0 };
@@ -60,14 +65,19 @@ const KeywordFilters = (props: KeywordFilterProps) => {
    const countryOptions = useMemo(() => {
       const optionObject:{label:string, value:string}[] = [];
 
-      Object.keys(countries).forEach((countryISO:string) => {
-         if (!isConsole || (isConsole && SCcountries.includes(countryISO))) {
-            optionObject.push({ label: countries[countryISO][0], value: countryISO });
-         }
-      });
+      if (!isConsole) {
+         const allCountries = keywords.reduce((acc: string[], keyword) => [...acc, keyword.country], []).filter((t) => t && t.trim() !== '');
+         [...new Set(allCountries)].forEach((c) => optionObject.push({ label: countries[c][0], value: c }));
+      } else {
+         Object.keys(countries).forEach((countryISO:string) => {
+            if ((SCcountries.includes(countryISO))) {
+               optionObject.push({ label: countries[countryISO][0], value: countryISO });
+            }
+         });
+      }
 
       return optionObject;
-   }, [SCcountries, isConsole]);
+   }, [SCcountries, isConsole, keywords]);
 
    const sortOptionChoices: SelectionOption[] = [
       { value: 'pos_asc', label: 'Top Position' },
@@ -79,6 +89,17 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       { value: 'vol_asc', label: 'Lowest Search Volume' },
       { value: 'vol_desc', label: 'Highest Search Volume' },
    ];
+
+   const columnOptionChoices: {label: string, value: string, locked: boolean}[] = [
+      { value: 'Keyword', label: 'Keyword', locked: true },
+      { value: 'Position', label: 'Position', locked: true },
+      { value: 'URL', label: 'URL', locked: true },
+      { value: 'Updated', label: 'Updated', locked: true },
+      { value: 'Best', label: 'Best', locked: false },
+      { value: 'History', label: 'History', locked: false },
+      { value: 'Volume', label: 'Volume', locked: false },
+      { value: 'Search Console', label: 'Search Console', locked: false },
+   ];
    if (integratedConsole) {
       sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
       sortOptionChoices.push({ value: 'imp_asc', label: 'Least Viewed' });
@@ -185,6 +206,43 @@ const KeywordFilters = (props: KeywordFilterProps) => {
                      </ul>
                   )}
                </div>
+               {!isConsole && (
+                  <div className='relative'>
+                  <button
+                  data-testid="columns_button"
+                  className={`px-2 py-1 rounded ${columnOptions ? ' bg-indigo-100 text-blue-700' : ''}`}
+                  title='Show/Hide Columns'
+                  onClick={() => showColumnOptions(!columnOptions)}
+                  >
+                     <Icon type='eye-closed' size={18} />
+                  </button>
+                  {columnOptions && (
+                     <ul
+                     data-testid="sort_options"
+                     className='sort_options mt-2 border absolute w-48 min-w-[0] right-0 rounded-lg
+                     max-h-96 bg-white z-[9999] overflow-y-auto styled-scrollbar border-gray-200 '>
+                        {columnOptionChoices.map(({ value, label, locked }) => {
+                           return <li
+                                    key={value}
+                                    className={sortItemStyle(value) + (locked ? 'bg-gray-50 cursor-not-allowed pointer-events-none' : '') }
+                                    onClick={() => { if (updateColumns) { updateColumns(value); } showColumnOptions(false); }}
+                                    >
+                                       <span className={' inline-block px-[3px] border border-gray-200  rounded-[4px] w-5'}>
+                                          <Icon
+                                          title={locked ? 'Cannot be Hidden' : ''}
+                                          type={locked ? 'lock' : 'check'}
+                                          color={!tableColumns.includes(value) && !locked ? 'transparent' : '#999' }
+                                          size={12}
+                                          />
+                                        </span>
+                                       {' '}{label}
+
+                                    </li>;
+                        })}
+                     </ul>
+                  )}
+               </div>
+               )}
          </div>
       </div>
    );

components/keywords/KeywordsTable.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useMemo } from 'react';
+import React, { useState, useMemo, useCallback, useRef, useEffect } from 'react';
 import { Toaster } from 'react-hot-toast';
 import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
 import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/client/sortFilter';
@@ -12,6 +12,8 @@ import KeywordTagManager from './KeywordTagManager';
 import AddTags from './AddTags';
 import useWindowResize from '../../hooks/useWindowResize';
 import useIsMobile from '../../hooks/useIsMobile';
+import { useUpdateSettings } from '../../services/settings';
+import { defaultSettings } from '../settings/Settings';
 
 type KeywordsTableProps = {
    domain: DomainType | null,
@@ -20,10 +22,12 @@ type KeywordsTableProps = {
    showAddModal: boolean,
    setShowAddModal: Function,
    isConsoleIntegrated: boolean,
+   settings?: SettingsType
 }
 
 const KeywordsTable = (props: KeywordsTableProps) => {
-   const { keywords = [], isLoading = true, isConsoleIntegrated = false } = props;
+   const titleColumnRef = useRef(null);
+   const { keywords = [], isLoading = true, isConsoleIntegrated = false, settings } = props;
    const showSCData = isConsoleIntegrated;
    const [device, setDevice] = useState<string>('desktop');
    const [selectedKeywords, setSelectedKeywords] = useState<number[]>([]);
@@ -36,11 +40,27 @@ const KeywordsTable = (props: KeywordsTableProps) => {
    const [sortBy, setSortBy] = useState<string>('date_asc');
    const [scDataType, setScDataType] = useState<string>('threeDays');
    const [showScDataTypes, setShowScDataTypes] = useState<boolean>(false);
+   const [maxTitleColumnWidth, setMaxTitleColumnWidth] = useState(235);
    const { mutate: deleteMutate } = useDeleteKeywords(() => {});
    const { mutate: favoriteMutate } = useFavKeywords(() => {});
    const { mutate: refreshMutate } = useRefreshKeywords(() => {});
    const [isMobile] = useIsMobile();
-   useWindowResize(() => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400)));
+
+   useWindowResize(() => {
+      setSCListHeight(window.innerHeight - (isMobile ? 200 : 400));
+      if (titleColumnRef.current) {
+         setMaxTitleColumnWidth((titleColumnRef.current as HTMLElement).clientWidth);
+      }
+   });
+
+   useEffect(() => {
+      if (titleColumnRef.current) {
+         setMaxTitleColumnWidth((titleColumnRef.current as HTMLElement).clientWidth);
+      }
+   }, [titleColumnRef]);
+
+   const tableColumns = settings?.keywordsColumns || ['Best', 'History', 'Volume', 'Search Console'];
+   const { mutate: updateMutate, isLoading: isUpdatingSettings } = useUpdateSettings(() => console.log(''));
 
    const scDataObject:{ [k:string] : string} = {
       threeDays: 'Last Three Days',
@@ -71,6 +91,16 @@ const KeywordsTable = (props: KeywordsTableProps) => {
       }
       setSelectedKeywords(updatedSelectd);
    };
+
+   const updateColumns = (column:string) => {
+      const newColumns = tableColumns.includes(column) ? tableColumns.filter((col) => col !== column) : [...tableColumns, column];
+      updateMutate({ ...defaultSettings, ...settings, keywordsColumns: newColumns });
+   };
+
+   const shouldHideColumn = useCallback((col:string) => {
+      return settings?.keywordsColumns && !settings?.keywordsColumns.includes(col) ? 'lg:hidden' : '';
+   }, [settings?.keywordsColumns]);
+
    const Row = ({ data, index, style }:ListChildComponentProps) => {
       const keyword = data[index];
       return (
@@ -89,6 +119,8 @@ const KeywordsTable = (props: KeywordsTableProps) => {
          lastItem={index === (processedKeywords[device].length - 1)}
          showSCData={showSCData}
          scDataType={scDataType}
+         tableColumns={tableColumns}
+         maxTitleColumnWidth={maxTitleColumnWidth}
          />
       );
    };
@@ -136,15 +168,19 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                   keywords={keywords}
                   device={device}
                   setDevice={setDevice}
+                  updateColumns={updateColumns}
+                  tableColumns={tableColumns}
                   integratedConsole={isConsoleIntegrated}
                />
             )}
-            <div className={`domkeywordsTable domkeywordsTable--keywords ${showSCData ? 'domkeywordsTable--hasSC' : ''} 
+            <div className={`domkeywordsTable domkeywordsTable--keywords 
+            ${showSCData && tableColumns.includes('Search Console') ? 'domkeywordsTable--hasSC' : ''} 
                styled-scrollbar w-full overflow-auto min-h-[60vh]`}>
                <div className=' lg:min-w-[800px]'>
                   <div className={`domKeywords_head domKeywords_head--${sortBy} hidden lg:flex p-3 px-6 bg-[#FCFCFF]
                    text-gray-600 justify-between items-center font-semibold border-y`}>
-                     <span className='domKeywords_head_keyword flex-1 basis-[4rem] w-auto '>
+                     <span ref={titleColumnRef} className={`domKeywords_head_keyword flex-1 basis-[4rem] w-auto lg:flex-1 
+                        ${showSCData && tableColumns.includes('Search Console') ? 'lg:basis-20' : 'lg:basis-10'} lg:w-auto lg:flex lg:items-center `}>
                      {processedKeywords[device].length > 0 && (
                         <button
                            className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px]  border border-slate-300 
@@ -154,16 +190,20 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                               <Icon type="check" size={10} />
                         </button>
                      )}
-                        Keyword
+                  {/* ${showSCData ? 'lg:min-w-[220px]' : 'lg:min-w-[280px]'} */}
+                        <span className={`inline-block lg:flex lg:items-center 
+                           ${showSCData && tableColumns.includes('Search Console') ? 'lg:max-w-[235px]' : ''}`}>
+                           Keyword
+                        </span>
                      </span>
                      <span className='domKeywords_head_position flex-1 basis-24 grow-0 text-center'>Position</span>
-                     <span className='domKeywords_head_best flex-1 basis-16 grow-0 text-center'>Best</span>
-                     <span className='domKeywords_head_history flex-1 basis-20 grow-0'>History (7d)</span>
-                     <span className='domKeywords_head_volume flex-1 basis-24 grow-0 text-center'>Volume</span>
+                     <span className={`domKeywords_head_best flex-1 basis-16 grow-0 text-center  ${shouldHideColumn('Best')}`}>Best</span>
+                     <span className={`domKeywords_head_history flex-1 basis-20 grow-0  ${shouldHideColumn('History')}`}>History (7d)</span>
+                     <span className={`domKeywords_head_volume flex-1 basis-24 grow-0 text-center ${shouldHideColumn('Volume')}`}>Volume</span>
                      <span className='domKeywords_head_url flex-1'>URL</span>
-                     <span className='domKeywords_head_updated flex-1 relative left-3'>Updated</span>
-                     {showSCData && (
-                        <div className='domKeywords_head_sc flex-1 min-w-[170px] mr-7 text-center'>
+                     <span className='domKeywords_head_updated flex-1 relative left-3 max-w-[150px]'>Updated</span>
+                     {showSCData && tableColumns.includes('Search Console') && (
+                        <div className='domKeywords_head_sc flex-1 min-w-[170px] lg:max-w-[170px] mr-7 text-center'>
                            {/* Search Console */}
                            <div>
                               <div

components/settings/NotificationSettings.tsx

@@ -89,6 +89,15 @@ const NotificationSettings = ({ settings, settingsError, updateSettings }:Notifi
                         onChange={(value:string) => updateSettings('notification_email_from', value)}
                         />
                   </div>
+                  <div className="settings__section__input mb-5">
+                        <InputField
+                        label='Email From Name'
+                        hasError={settingsError?.type === 'no_smtp_from'}
+                        value={settings?.notification_email_from_name || 'Serpbear'}
+                        placeholder="Serpbear"
+                        onChange={(value:string) => updateSettings('notification_email_from_name', value)}
+                        />
+                  </div>
                </>
             )}
 

components/settings/Settings.tsx

@@ -17,7 +17,7 @@ type SettingsError = {
    msg: string
 }
 
-const defaultSettings: SettingsType = {
+export const defaultSettings: SettingsType = {
    scraper_type: 'none',
    scrape_delay: 'none',
    scrape_retry: false,
@@ -28,9 +28,11 @@ const defaultSettings: SettingsType = {
    smtp_username: '',
    smtp_password: '',
    notification_email_from: '',
+   notification_email_from_name: 'SerpBear',
    search_console: true,
    search_console_client_email: '',
    search_console_private_key: '',
+   keywordsColumns: ['Best', 'History', 'Volume', 'Search Console'],
 };
 
 const Settings = ({ closeSettings }:SettingsProps) => {

email/email.html

@@ -442,6 +442,7 @@
                               <tr align="left">
                                  <th>Keyword</th>
                                  <th>Position</th>
+                                 <th>Best</th>
                                  <th>Updated</th>
                                </tr>
                                {{keywordsTable}}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "serpbear",
-  "version": "2.0.0",
+  "version": "2.0.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "serpbear",
-      "version": "2.0.0",
+      "version": "2.0.3",
       "dependencies": {
         "@googleapis/searchconsole": "^1.0.0",
         "@isaacs/ttlcache": "^1.4.1",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "2.0.0",
+  "version": "2.0.3",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -63,8 +63,8 @@
     "@types/jsonwebtoken": "^8.5.9",
     "@types/node": "18.11.0",
     "@types/nodemailer": "^6.4.6",
-    "@types/react": "18.0.21",
-    "@types/react-dom": "18.0.6",
+    "@types/react": "18.2.0",
+    "@types/react-dom": "18.2.0",
     "@types/react-timeago": "^4.1.3",
     "@types/react-window": "^1.8.5",
     "autoprefixer": "^10.4.12",

pages/api/adwords.ts

@@ -29,7 +29,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 const getAdwordsRefreshToken = async (req: NextApiRequest, res: NextApiResponse<string>) => {
    try {
       const code = (req.query.code as string);
-      // console.log('code :', code);
+      const https = req.headers.host?.includes('localhost:') ? 'http://' : 'https://';
+      const redirectURL = `${https}${req.headers.host}/api/adwords`;
+
       if (code) {
          try {
             const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
@@ -37,7 +39,6 @@ const getAdwordsRefreshToken = async (req: NextApiRequest, res: NextApiResponse<
             const cryptr = new Cryptr(process.env.SECRET as string);
             const adwords_client_id = settings.adwords_client_id ? cryptr.decrypt(settings.adwords_client_id) : '';
             const adwords_client_secret = settings.adwords_client_secret ? cryptr.decrypt(settings.adwords_client_secret) : '';
-            const redirectURL = `${process.env.NEXT_PUBLIC_APP_URL}/api/adwords`;
             const oAuth2Client = new OAuth2Client(adwords_client_id, adwords_client_secret, redirectURL);
             const r = await oAuth2Client.getToken(code);
             if (r?.tokens?.refresh_token) {
@@ -45,17 +46,20 @@ const getAdwordsRefreshToken = async (req: NextApiRequest, res: NextApiResponse<
                await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify({ ...settings, adwords_refresh_token }), { encoding: 'utf-8' });
                return res.status(200).send('Google Ads Intergrated Successfully! You can close this window.');
             }
-            return res.status(200).send('Error Getting the Google Ads Refresh Token. Please Try Again!');
-         } catch (error) {
-            console.log('[Error] Getting Google Ads Refresh Token!');
-            console.log('error :', error);
-            return res.status(200).send('Error Saving the Google Ads Refresh Token. Please Try Again!');
+            return res.status(400).send('Error Getting the Google Ads Refresh Token. Please Try Again!');
+         } catch (error:any) {
+            let errorMsg = error?.response?.data?.error;
+            if (errorMsg.includes('redirect_uri_mismatch')) {
+               errorMsg += ` Redirected URL: ${redirectURL}`;
+            }
+            console.log('[Error] Getting Google Ads Refresh Token! Reason: ', errorMsg);
+            return res.status(400).send(`Error Saving the Google Ads Refresh Token ${errorMsg ? `. Details: ${errorMsg}` : ''}. Please Try Again!`);
          }
       } else {
-         return res.status(200).send('No Code Provided By Google. Please Try Again!');
+         return res.status(400).send('No Code Provided By Google. Please Try Again!');
       }
    } catch (error) {
-      console.log('[ERROR] CRON Refreshing Keywords: ', error);
+      console.log('[ERROR] Getting Google Ads Refresh Token: ', error);
       return res.status(400).send('Error Getting Google Ads Refresh Token. Please Try Again!');
    }
 };

pages/api/keywords.ts

@@ -46,9 +46,11 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
    if (!req.query.domain && typeof req.query.domain !== 'string') {
       return res.status(400).json({ error: 'Domain is Required!' });
    }
+   const settings = await getAppSettings();
    const domain = (req.query.domain as string);
    const integratedSC = process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL;
-   const domainSCData = integratedSC ? await readLocalSCData(domain) : false;
+   const { search_console_client_email, search_console_private_key } = settings;
+   const domainSCData = integratedSC || (search_console_client_email && search_console_private_key) ? await readLocalSCData(domain) : false;
 
    try {
       const allKeywords:Keyword[] = await Keyword.findAll({ where: { domain } });

pages/api/notify.ts

@@ -62,9 +62,10 @@ const sendNotificationEmail = async (domain: Domain, settings: SettingsType) =>
       smtp_password = '',
       notification_email = '',
       notification_email_from = '',
+      notification_email_from_name = 'SerpBear',
      } = settings;
 
-   const fromEmail = `SerpBear <${notification_email_from || 'no-reply@serpbear.com'}>`;
+   const fromEmail = `${notification_email_from_name} <${notification_email_from || 'no-reply@serpbear.com'}>`;
    const mailerSettings:any = { host: smtp_server, port: parseInt(smtp_port, 10) };
    if (smtp_username || smtp_password) {
       mailerSettings.auth = {};
@@ -77,7 +78,7 @@ const sendNotificationEmail = async (domain: Domain, settings: SettingsType) =>
    const domainKeywords:Keyword[] = await Keyword.findAll(query);
    const keywordsArray = domainKeywords.map((el) => el.get({ plain: true }));
    const keywords: KeywordType[] = parseKeywords(keywordsArray);
-   const emailHTML = await generateEmail(domainName, keywords);
+   const emailHTML = await generateEmail(domainName, keywords, settings);
    await transporter.sendMail({
       from: fromEmail,
       to: domain.notification_emails || notification_email,

pages/api/settings.ts

@@ -119,6 +119,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          notification_interval: 'never',
          notification_email: '',
          notification_email_from: '',
+         notification_email_from_name: 'SerpBear',
          smtp_server: '',
          smtp_port: '',
          smtp_username: '',
@@ -128,6 +129,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          search_console: true,
          search_console_client_email: '',
          search_console_private_key: '',
+         keywordsColumns: ['Best', 'History', 'Volume', 'Search Console'],
       };
       const otherSettings = {
          available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),

pages/domain/[slug]/index.tsx

@@ -80,6 +80,7 @@ const SingleDomain: NextPage = () => {
                showAddModal={showAddKeywords}
                setShowAddModal={setShowAddKeywords}
                isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) || domainHasScAPI }
+               settings={appSettings}
                />
             </div>
          </div>

pages/domain/console/[slug]/index.tsx

@@ -30,7 +30,49 @@ const DiscoverPage: NextPage = () => {
    const { data: keywordsData, isLoading: keywordsLoading, isFetching } = useFetchSCKeywords(router, !!(domainsData?.domains?.length) && scConnected);
 
    const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
-   const theKeywords: SearchAnalyticsItem[] = keywordsData?.data && keywordsData.data[scDateFilter] ? keywordsData.data[scDateFilter] : [];
+   const theKeywords: SearchAnalyticsItem[] = useMemo(() => {
+      return keywordsData?.data && keywordsData.data[scDateFilter] ? keywordsData.data[scDateFilter] : [];
+   }, [keywordsData, scDateFilter]);
+
+   const theKeywordsCount = useMemo(() => {
+      return theKeywords.reduce<Map<string, number>>((r, o) => {
+         const key = `${o.device}-${o.country}-${o.keyword}`;
+         const item = r.get(key) || 0;
+         return r.set(key, item + 1);
+      }, new Map()) || [];
+   }, [theKeywords]);
+
+   const theKeywordsReduced : SearchAnalyticsItem[] = useMemo(() => {
+      return [...theKeywords.reduce<Map<string, SearchAnalyticsItem>>((r, o) => {
+         const key = `${o.device}-${o.country}-${o.keyword}`;
+         const item = r.get(key) || { ...o,
+            ...{
+            clicks: 0,
+            impressions: 0,
+            ctr: 0,
+            position: 0,
+            },
+         };
+         item.clicks += o.clicks;
+         item.impressions += o.impressions;
+         item.ctr = o.ctr + item.ctr;
+         item.position = o.position + item.position;
+         return r.set(key, item);
+      }, new Map()).values()];
+   }, [theKeywords]);
+
+   const theKeywordsGrouped : SearchAnalyticsItem[] = useMemo(() => {
+      return [...theKeywordsReduced.map<SearchAnalyticsItem>((o: SearchAnalyticsItem) => {
+         const key = `${o.device}-${o.country}-${o.keyword}`;
+         const count = theKeywordsCount?.get(key) || 0;
+         return { ...o,
+            ...{
+            ctr: Math.round((o.ctr / count) * 100) / 100,
+            position: Math.round(o.position / count),
+            },
+         };
+      })];
+   }, [theKeywordsReduced, theKeywordsCount]);
 
    const activDomain: DomainType|null = useMemo(() => {
       let active:DomainType|null = null;
@@ -62,7 +104,7 @@ const DiscoverPage: NextPage = () => {
                   domains={theDomains}
                   showAddModal={() => console.log('XXXXX')}
                   showSettingsModal={setShowDomainSettings}
-                  exportCsv={() => exportCSV(theKeywords, activDomain.domain, scDateFilter)}
+                  exportCsv={() => exportCSV(theKeywordsGrouped, activDomain.domain, scDateFilter)}
                   scFilter={scDateFilter}
                   setScFilter={(item:string) => setSCDateFilter(item)}
                   />
@@ -71,7 +113,7 @@ const DiscoverPage: NextPage = () => {
                <SCKeywordsTable
                isLoading={keywordsLoading || isFetching}
                domain={activDomain}
-               keywords={theKeywords}
+               keywords={theKeywordsGrouped}
                isConsoleIntegrated={scConnected || domainHasScAPI}
                />
             </div>

scrapers/index.ts

@@ -7,6 +7,7 @@ import proxy from './services/proxy';
 import searchapi from './services/searchapi';
 import valueSerp from './services/valueserp';
 import serper from './services/serper';
+import hasdata from './services/hasdata';
 
 export default [
    scrapingRobot,
@@ -18,4 +19,5 @@ export default [
    searchapi,
    valueSerp,
    serper,
+   hasdata,
 ];

scrapers/services/hasdata.ts

@@ -0,0 +1,45 @@
+import countries from '../../utils/countries';
+
+interface HasDataResult {
+   title: string,
+   link: string,
+   position: number,
+}
+
+const hasdata:ScraperSettings = {
+   id: 'hasdata',
+   name: 'HasData',
+   website: 'hasdata.com',
+   allowsCity: true,
+   headers: (keyword, settings) => {
+      return {
+         'Content-Type': 'application/json',
+         'x-api-key': settings.scaping_api,
+      };
+   },
+   scrapeURL: (keyword, settings) => {
+      const country = keyword.country || 'US';
+      const countryName = countries[country][0];
+      const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
+   },
+   resultObjectKey: 'organicResults',
+   
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[];
+
+      for (const { link, title, position } of results) {
+         if (title && link) {
+            extractedResult.push({
+               title,
+               url: link,
+               position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default hasdata;

styles/globals.css

@@ -8,6 +8,10 @@ body {
    background-color: #f8f9ff;
 }
 
+.max-w-7xl {
+   max-width: 90rem;
+}
+
 .domKeywords {
    /* min-height: 70vh; */
    border-color: #e9ebff;
@@ -301,6 +305,13 @@ body {
    background-color: #f9ded7;
 }
 
+@media screen and (max-width: 760px){
+   .keyword_best:before{
+      content: "Best: ";
+   }
+}
+
+
 /* Disable LastPass Icon for Secret Field */
 [autocomplete="off"] + div[data-lastpass-icon-root="true"], [autocomplete="off"] + div[data-lastpass-infield="true"] {
    display: none;

types.d.ts

@@ -85,6 +85,7 @@ type SettingsType = {
    notification_interval: string,
    notification_email: string,
    notification_email_from: string,
+   notification_email_from_name: string,
    smtp_server: string,
    smtp_port: string,
    smtp_username?: string,
@@ -105,6 +106,7 @@ type SettingsType = {
    adwords_refresh_token?: string,
    adwords_developer_token?: string,
    adwords_account_id?: string,
+   keywordsColumns: string[]
 }
 
 type KeywordSCDataChild = {

utils/adwords.ts

@@ -271,7 +271,7 @@ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error
    // Generate Access Token
    let accessToken = '';
    const cachedAccessToken:string|false|undefined = memoryCache.get('adwords_token');
-   if (cachedAccessToken && !test) {
+   if (cachedAccessToken) {
       accessToken = cachedAccessToken;
    } else {
       accessToken = await getAdwordsAccessToken(credentials);

utils/client/exportcsv.ts

@@ -9,7 +9,7 @@ import countries from '../countries';
 const exportCSV = (keywords: KeywordType[] | SCKeywordType[], domain:string, scDataDuration = 'lastThreeDays') => {
    if (!keywords || (keywords && Array.isArray(keywords) && keywords.length === 0)) { return; }
    const isSCKeywords = !!(keywords && keywords[0] && keywords[0].uid);
-   let csvHeader = 'ID,Keyword,Position,URL,Country,Device,Updated,Added,Tags\r\n';
+   let csvHeader = 'ID,Keyword,Position,URL,Country,City,Device,Updated,Added,Tags\r\n';
    let csvBody = '';
    let fileName = `${domain}-keywords_serp.csv`;
 
@@ -26,9 +26,9 @@ const exportCSV = (keywords: KeywordType[] | SCKeywordType[], domain:string, scD
       });
    } else {
       keywords.forEach((keywordData) => {
-         const { ID, keyword, position, url, country, device, lastUpdated, added, tags } = keywordData as KeywordType;
+         const { ID, keyword, position, url, country, city, device, lastUpdated, added, tags } = keywordData as KeywordType;
          // eslint-disable-next-line max-len
-         csvBody += `${ID}, ${keyword}, ${position === 0 ? '-' : position}, ${url || '-'}, ${countries[country][0]}, ${device}, ${lastUpdated}, ${added}, ${tags.join(',')}\r\n`;
+         csvBody += `${ID}, ${keyword}, ${position === 0 ? '-' : position}, ${url || '-'}, ${countries[country][0]}, ${city || '-'}, ${device}, ${lastUpdated}, ${added}, ${tags.join(',')}\r\n`;
       });
    }
 

utils/generateEmail.ts

@@ -68,13 +68,26 @@ const getPositionChange = (history:KeywordHistory, position:number) : number =>
    return status;
 };
 
+const getBestKeywordPosition = (history: KeywordHistory) => {
+   let bestPos;
+   if (Object.keys(history).length > 0) {
+      const historyArray = Object.keys(history).map((itemID) => ({ date: itemID, position: history[itemID] }))
+          .sort((a, b) => a.position - b.position).filter((el) => (el.position > 0));
+      if (historyArray[0]) {
+         bestPos = { ...historyArray[0] };
+      }
+   }
+
+   return bestPos?.position || '-';
+};
+
 /**
  * Generate the Email HTML based on given domain name and its keywords
  * @param {string} domainName - Keywords to scrape
  * @param {keywords[]} keywords - Keywords to scrape
  * @returns {Promise}
  */
-const generateEmail = async (domainName:string, keywords:KeywordType[]) : Promise<string> => {
+const generateEmail = async (domainName:string, keywords:KeywordType[], settings: SettingsType) : Promise<string> => {
    const emailTemplate = await readFile(path.join(__dirname, '..', '..', '..', '..', 'email', 'email.html'), { encoding: 'utf-8' });
    const currentDate = dayjs(new Date()).format('MMMM D, YYYY');
    const keywordsCount = keywords.length;
@@ -88,7 +101,7 @@ const generateEmail = async (domainName:string, keywords:KeywordType[]) : Promis
       const positionChange = getPositionChange(keyword.history, keyword.position);
       const deviceIconImg = keyword.device === 'desktop' ? desktopIcon : mobileIcon;
       const countryFlag = `<img class="flag" src="https://flagcdn.com/w20/${keyword.country.toLowerCase()}.png" alt="${keyword.country}" title="${keyword.country}" />`;
-      const deviceIcon = `<img class="device" src="${deviceIconImg}" alt="${keyword.device}" title="${keyword.device}" />`;
+      const deviceIcon = `<img class="device" src="${deviceIconImg}" alt="${keyword.device}" title="${keyword.device}" width="18" height="18" />`;
 
       if (positionChange > 0) { positionChangeIcon = '<span style="color:#5ed7c3;">▲</span>'; improved += 1; }
       if (positionChange < 0) { positionChangeIcon = '<span style="color:#fca5a5;">▼</span>'; declined += 1; }
@@ -97,6 +110,7 @@ const generateEmail = async (domainName:string, keywords:KeywordType[]) : Promis
       keywordsTable += `<tr class="keyword">
                            <td>${countryFlag} ${deviceIcon} ${keyword.keyword}</td>
                            <td>${keyword.position}${posChangeIcon}</td>
+                           <td>${getBestKeywordPosition(keyword.history)}</td>
                            <td>${timeSince(new Date(keyword.lastUpdated).getTime() / 1000)}</td>
                         </tr>`;
    });
@@ -104,7 +118,7 @@ const generateEmail = async (domainName:string, keywords:KeywordType[]) : Promis
    const stat = `${improved > 0 ? `${improved} Improved` : ''} 
                   ${improved > 0 && declined > 0 ? ', ' : ''} ${declined > 0 ? `${declined} Declined` : ''}`;
    const updatedEmail = emailTemplate
-         .replace('{{logo}}', `<img class="logo_img" src="${serpBearLogo}" alt="SerpBear" />`)
+         .replace('{{logo}}', `<img class="logo_img" src="${serpBearLogo}" alt="SerpBear" width="24" height="24" />`)
          .replace('{{currentDate}}', currentDate)
          .replace('{{domainName}}', domainName)
          .replace('{{keywordsCount}}', keywordsCount.toString())
@@ -113,7 +127,8 @@ const generateEmail = async (domainName:string, keywords:KeywordType[]) : Promis
          .replace('{{stat}}', stat)
          .replace('{{preheader}}', stat);
 
-      const isConsoleIntegrated = !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL);
+      const isConsoleIntegrated = !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL)
+      || (settings.search_console_client_email && settings.search_console_private_key);
       const htmlWithSCStats = isConsoleIntegrated ? await generateGoogeleConsoleStats(domainName) : '';
       const emailHTML = updatedEmail.replace('{{SCStatsTable}}', htmlWithSCStats);
 
@@ -172,7 +187,7 @@ const generateGoogeleConsoleStats = async (domainName:string): Promise<string> =
       let htmlWithSCStats = `<table role="presentation" border="0" cellpadding="0" cellspacing="0" class="console_table">
                               <tr>
                                  <td style="font-weight:bold;">
-                                 <img class="google_icon" src="${googleIcon}" alt="Google"> Google Search Console Stats</h3>
+                                 <img class="google_icon" src="${googleIcon}" alt="Google" width="13" height="13"> Google Search Console Stats</h3>
                                  </td>
                                  <td class="stat" align="right" style="font-size: 12px;">
                                  ${startDate.getDate()} ${months[startDate.getMonth()]} -  ${endDate.getDate()} ${months[endDate.getMonth()]} 

utils/refresh.ts

@@ -72,14 +72,13 @@ export const updateKeywordPosition = async (keywordRaw:Keyword, udpatedkeyword:
 
       if (udpatedkeyword && keyword) {
          const newPos = udpatedkeyword.position;
-         const newPosition = newPos !== 0 ? newPos : keyword.position;
          const { history } = keyword;
          const theDate = new Date();
          const dateKey = `${theDate.getFullYear()}-${theDate.getMonth() + 1}-${theDate.getDate()}`;
-         history[dateKey] = newPosition;
+         history[dateKey] = newPos;
 
          const updatedVal = {
-            position: newPosition,
+            position: newPos,
             updating: false,
             url: udpatedkeyword.url,
             lastResult: udpatedkeyword.result,