chore(release): 0.2.4

fixes: 77, 80

CHANGELOG.md

@@ -2,6 +2,42 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.2.4](https://github.com/towfiqi/serpbear/compare/v0.2.3...v0.2.4) (2023-02-15)
+
+
+### Features
+
+* Keyword ranking pages can now be clicked. ([c5af94a](https://github.com/towfiqi/serpbear/commit/c5af94a1469713ed4092253d26953ee0ed28c25d))
+
+
+### Bug Fixes
+
+* Fixes broken Login on windows ([c406588](https://github.com/towfiqi/serpbear/commit/c406588953035e4177a64011c13eb0e3aedffe89))
+* Fixes Node Cron memory leak issue. ([3c48d13](https://github.com/towfiqi/serpbear/commit/3c48d130b6f229a4ac27ec43ef1ea3a6640cecf6))
+
+### [0.2.3](https://github.com/towfiqi/serpbear/compare/v0.2.2...v0.2.3) (2023-01-12)
+
+
+### Features
+
+* Ability to tag multiple keywords at once ([9e9dad7](https://github.com/towfiqi/serpbear/commit/9e9dad7631691b2a836fdd4c522b1f933b17e285)), closes [#54](https://github.com/towfiqi/serpbear/issues/54)
+* Set USERNAME as well as USER variable ([b50733d](https://github.com/towfiqi/serpbear/commit/b50733defc2c06e0f92ca3e88fd1f74684eee9c0))
+
+
+### Bug Fixes
+
+* Fixes Position and View Sort. ([8139e39](https://github.com/towfiqi/serpbear/commit/8139e399c13ab8be767facef9a19c67dec06ed64)), closes [#46](https://github.com/towfiqi/serpbear/issues/46)
+* Fixes wrong CTR value for Search Console Data ([cb24696](https://github.com/towfiqi/serpbear/commit/cb24696a1f47b02a11c68cd1c673ea8b1bacd144)), closes [#48](https://github.com/towfiqi/serpbear/issues/48)
+* Mobile Keyword Scraping not working. ([a1108d2](https://github.com/towfiqi/serpbear/commit/a1108d240ea38ab0886ef3722b0c937ec5a45591)), closes [#58](https://github.com/towfiqi/serpbear/issues/58)
+* ScrapingAnt Mobile Keyword Scrape not working ([acc0b39](https://github.com/towfiqi/serpbear/commit/acc0b39d80d4f9371967a0d425ed205c5d866eea))
+
+### [0.2.2](https://github.com/towfiqi/serpbear/compare/v0.2.1...v0.2.2) (2022-12-25)
+
+
+### Bug Fixes
+
+* Fixes bug that prevents Saving API settings ([123ad81](https://github.com/towfiqi/serpbear/commit/123ad81dae10aa28848148d0f3da5cf1f7de7c57)), closes [#45](https://github.com/towfiqi/serpbear/issues/45)
+
 ### [0.2.1](https://github.com/towfiqi/serpbear/compare/v0.2.0...v0.2.1) (2022-12-24)
 
 ## [0.2.0](https://github.com/towfiqi/serpbear/compare/v0.1.7...v0.2.0) (2022-12-21)

Dockerfile

@@ -29,7 +29,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/cron.js ./
 COPY --from=builder --chown=nextjs:nodejs /app/email ./email
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv node-cron @googleapis/searchconsole
+RUN npm i cryptr dotenv croner @googleapis/searchconsole
 RUN npm i -g concurrently
 
 USER nextjs

components/keywords/AddTags.tsx

@@ -0,0 +1,60 @@
+import { useState } from 'react';
+import { useUpdateKeywordTags } from '../../services/keywords';
+import Icon from '../common/Icon';
+import Modal from '../common/Modal';
+
+type AddTagsProps = {
+   keywords: KeywordType[],
+   closeModal: Function
+}
+
+const AddTags = ({ keywords = [], closeModal }: AddTagsProps) => {
+   const [tagInput, setTagInput] = useState('');
+   const [inputError, setInputError] = useState('');
+   const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
+
+   const addTag = () => {
+      if (keywords.length === 0) { return; }
+      if (!tagInput) {
+         setInputError('Please Insert a Tag!');
+         setTimeout(() => { setInputError(''); }, 3000);
+         return;
+      }
+
+      const tagsArray = tagInput.split(',').map((t) => t.trim());
+      const tagsPayload:any = {};
+      keywords.forEach((keyword:KeywordType) => {
+         tagsPayload[keyword.ID] = [...keyword.tags, ...tagsArray];
+      });
+      updateMutate({ tags: tagsPayload });
+   };
+
+   return (
+      <Modal closeModal={() => { closeModal(false); }} title={`Add New Tags to ${keywords.length} Selected Keyword`}>
+         <div className="relative">
+            {inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
+            <span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
+            <input
+               className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
+               placeholder='Insert Tags. eg: tag1, tag2'
+               value={tagInput}
+               onChange={(e) => setTagInput(e.target.value)}
+               onKeyDown={(e) => {
+                  if (e.code === 'Enter') {
+                    e.preventDefault();
+                    addTag();
+                  }
+               }}
+            />
+            <button
+            className=" absolute right-2 top-2 cursor-pointer rounded p-2 px-4 bg-indigo-600 text-white font-semibold text-sm"
+            onClick={addTag}>
+               Apply
+            </button>
+         </div>
+      </Modal>
+
+   );
+};
+
+export default AddTags;

components/keywords/Keyword.tsx

@@ -115,7 +115,10 @@ const Keyword = (props: KeywordProps) => {
          <div
          className={`keyword_url inline-block mt-4 mr-5 ml-5 lg:flex-1 text-gray-400 lg:m-0 max-w-[70px] 
          overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5`}>
-            <span className='mr-3 lg:hidden'><Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}</div>
+            <a href={url} target="_blank" rel="noreferrer"><span className='mr-3 lg:hidden'>
+               <Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}
+            </a>
+         </div>
          <div
          className='inline-block mt-[4] top-[-5px] relative lg:flex-1 lg:m-0'>
             <span className='mr-2 lg:hidden'><Icon type="clock" size={14} color="#999" /></span>

components/keywords/KeywordDetails.tsx

@@ -85,7 +85,7 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
                   <h3 className=' text-lg font-bold'>
                      <span title={countries[keyword.country][0]}
                      className={`fflag fflag-${keyword.country} w-[18px] h-[12px] mr-2`} /> {keyword.keyword}
-                     <span className='py-1 px-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
+                     <span className='py-1 px-2 ml-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
                   </h3>
                   <button
                   className='absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'

components/keywords/KeywordFilter.tsx

@@ -82,10 +82,10 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       { value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
    ];
    if (integratedConsole) {
-      sortOptionChoices.push({ value: 'imp_asc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
-      sortOptionChoices.push({ value: 'imp_desc', label: 'Least Viewed' });
-      sortOptionChoices.push({ value: 'visits_asc', label: 'Most Visited' });
-      sortOptionChoices.push({ value: 'visits_desc', label: 'Least Visited' });
+      sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
+      sortOptionChoices.push({ value: 'imp_asc', label: 'Least Viewed' });
+      sortOptionChoices.push({ value: 'visits_desc', label: 'Most Visited' });
+      sortOptionChoices.push({ value: 'visits_asc', label: 'Least Visited' });
    }
    if (isConsole) {
       sortOptionChoices.splice(2, 2);

components/keywords/KeywordTagManager.tsx

@@ -2,6 +2,7 @@ import { useState } from 'react';
 import { useUpdateKeywordTags } from '../../services/keywords';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
+import AddTags from './AddTags';
 
 type keywordTagManagerProps = {
    keyword: KeywordType|undefined,
@@ -10,9 +11,8 @@ type keywordTagManagerProps = {
 }
 
 const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
-   const [tagInput, setTagInput] = useState('');
-   const [inputError, setInputError] = useState('');
-   const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
+   const [showAddTag, setShowAddTag] = useState<boolean>(false);
+   const { mutate: updateMutate } = useUpdateKeywordTags(() => { });
 
    const removeTag = (tag:String) => {
       if (!keyword) { return; }
@@ -20,24 +20,6 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
       updateMutate({ tags: { [keyword.ID]: newTags } });
    };
 
-   const addTag = () => {
-      if (!keyword) { return; }
-      if (!tagInput) {
-         setInputError('Please Insert a Tag!');
-         setTimeout(() => { setInputError(''); }, 3000);
-         return;
-      }
-      if (keyword.tags.includes(tagInput)) {
-         setInputError('Tag Exist!');
-         setTimeout(() => { setInputError(''); }, 3000);
-         return;
-      }
-
-      console.log('New Tag: ', tagInput);
-      const newTags = [...keyword.tags, tagInput.trim()];
-      updateMutate({ tags: { [keyword.ID]: newTags } });
-   };
-
    return (
       <Modal closeModal={() => { closeModal(false); }} title={`Tags for Keyword "${keyword && keyword.keyword}"`}>
          <div className="text-sm my-8 ">
@@ -53,31 +35,27 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
                                  </button>
                            </li>;
                   })}
+                  <li className='inline-block py-1 px-1'>
+                     <button
+                     title='Add New Tag'
+                     className="cursor-pointer rounded p-1 px-3 bg-indigo-600 text-white font-semibold text-sm"
+                     onClick={() => setShowAddTag(true)}>+</button>
+                  </li>
                </ul>
             )}
             {keyword && keyword.tags.length === 0 && (
-               <div className="text-center w-full text-gray-500">No Tags Added to this Keyword.</div>
+               <div className="text-center w-full text-gray-500">
+                  No Tags Added to this Keyword. <button className=' text-indigo-600' onClick={() => setShowAddTag(true)}>+ Add Tag</button>
+               </div>
             )}
          </div>
-         <div className="relative">
-            {inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
-            <span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
-            <input
-               className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
-               placeholder='Insert Tags'
-               value={tagInput}
-               onChange={(e) => setTagInput(e.target.value)}
-               onKeyDown={(e) => {
-                  if (e.code === 'Enter') {
-                    e.preventDefault();
-                    addTag();
-                  }
-               }}
-            />
-            <button className=" absolute right-2 top-2 cursor-pointer rounded p-1 px-4 bg-blue-600 text-white font-bold" onClick={addTag}>+</button>
-         </div>
+         {showAddTag && keyword && (
+            <AddTags
+               keywords={[keyword]}
+               closeModal={() => setShowAddTag(false)}
+               />
+         )}
       </Modal>
-
    );
 };
 

components/keywords/KeywordsTable.tsx

@@ -10,6 +10,7 @@ import KeywordFilters from './KeywordFilter';
 import Modal from '../common/Modal';
 import { useDeleteKeywords, useFavKeywords, useRefreshKeywords } from '../../services/keywords';
 import KeywordTagManager from './KeywordTagManager';
+import AddTags from './AddTags';
 
 type KeywordsTableProps = {
    domain: DomainType | null,
@@ -28,6 +29,7 @@ const KeywordsTable = (props: KeywordsTableProps) => {
    const [showKeyDetails, setShowKeyDetails] = useState<KeywordType|null>(null);
    const [showRemoveModal, setShowRemoveModal] = useState<boolean>(false);
    const [showTagManager, setShowTagManager] = useState<null|number>(null);
+   const [showAddTags, setShowAddTags] = useState<boolean>(false);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
    const [sortBy, setSortBy] = useState<string>('date_asc');
    const [scDataType, setScDataType] = useState<string>('threeDays');
@@ -79,7 +81,7 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                         className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
                         onClick={() => { refreshMutate({ ids: selectedKeywords }); setSelectedKeywords([]); }}
                         >
-                           <span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keyword
+                           <span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keywords
                         </a>
                      </li>
                      <li className='inline-block mr-4'>
@@ -87,7 +89,14 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                         className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
                         onClick={() => setShowRemoveModal(true)}
                         >
-                           <span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keyword</a>
+                           <span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keywords</a>
+                     </li>
+                     <li className='inline-block mr-4'>
+                        <a
+                        className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
+                        onClick={() => setShowAddTags(true)}
+                        >
+                           <span className=' bg-green-100 text-green-500  px-1 rounded'><Icon type="tags" size={14} /></span> Tag Keywords</a>
                      </li>
                   </ul>
                </div>
@@ -222,6 +231,12 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                closeModal={() => setShowTagManager(null)}
                />
          )}
+         {showAddTags && (
+            <AddTags
+               keywords={keywords.filter((k) => selectedKeywords.includes(k.ID))}
+               closeModal={() => setShowAddTags(false)}
+               />
+         )}
          <Toaster position='bottom-center' containerClassName="react_toaster" />
       </div>
    );

components/keywords/SCKeywordsTable.tsx

@@ -26,7 +26,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
    const [device, setDevice] = useState<string>('desktop');
    const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
-   const [sortBy, setSortBy] = useState<string>('imp_asc');
+   const [sortBy, setSortBy] = useState<string>('imp_desc');
    const [isMobile, setIsMobile] = useState<boolean>(false);
    const [SCListHeight, setSCListHeight] = useState(500);
    const { keywordsData } = useFetchKeywords(router);

components/settings/Settings.tsx

@@ -63,21 +63,20 @@ const Settings = ({ closeSettings }:SettingsProps) => {
 
    const performUpdate = () => {
       let error: null|SettingsError = null;
-      if (settings.notification_interval !== 'never') {
+      const { notification_interval, notification_email, notification_email_from, scraper_type, smtp_port, smtp_server, scaping_api } = settings;
+      if (notification_interval !== 'never') {
          if (!settings.notification_email) {
             error = { type: 'no_email', msg: 'Insert a Valid Email address' };
          }
-         if (settings.notification_email
-            && (!settings.smtp_port || !settings.smtp_server
-               || !settings.notification_email_from)) {
+         if (notification_email && (!smtp_port || !smtp_server || !notification_email_from)) {
                let type = 'no_smtp_from';
-               if (!settings.smtp_port) { type = 'no_smtp_port'; }
-               if (!settings.smtp_server) { type = 'no_smtp_server'; }
+               if (!smtp_port) { type = 'no_smtp_port'; }
+               if (!smtp_server) { type = 'no_smtp_server'; }
                error = { type, msg: 'Insert SMTP Server details that will be used to send the emails.' };
          }
       }
 
-      if (settings?.scraper_type !== 'proxy' && settings?.scraper_type !== 'none') {
+      if (scraper_type !== 'proxy' && scraper_type !== 'none' && !scaping_api) {
          error = { type: 'no_api_key', msg: 'Insert a Valid API Key or Token for the Scraper Service.' };
       }
 

cron.js

@@ -1,7 +1,7 @@
 const Cryptr = require('cryptr');
 const { promises } = require('fs');
 const { readFile } = require('fs');
-const cron = require('node-cron');
+const Cron = require('croner');
 require('dotenv').config({ path: './.env.local' });
 
 const getAppSettings = async () => {
@@ -65,12 +65,12 @@ const generateCronTime = (interval) => {
 const runAppCronJobs = () => {
    // RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
    const scrapeCronTime = generateCronTime('daily');
-   cron.schedule(scrapeCronTime, () => {
+   Cron(scrapeCronTime, () => {
       // console.log('### Running Keyword Position Cron Job!');
       const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
       fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
       .then((res) => res.json())
-      .then((data) => console.log(data))
+      // .then((data) =>{ console.log(data)})
       .catch((err) => {
          console.log('ERROR Making Daily Scraper Cron Request..');
          console.log(err);
@@ -79,7 +79,7 @@ const runAppCronJobs = () => {
 
    // Run Failed scraping CRON (Every Hour)
    const failedCronTime = generateCronTime('hourly');
-   cron.schedule(failedCronTime, () => {
+   Cron(failedCronTime, () => {
       // console.log('### Retrying Failed Scrapes...');
 
       readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
@@ -104,7 +104,7 @@ const runAppCronJobs = () => {
    // Run Google Search Console Scraper Daily
    if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
       const searchConsoleCRONTime = generateCronTime('daily');
-      cron.schedule(searchConsoleCRONTime, () => {
+      Cron(searchConsoleCRONTime, () => {
          const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
          fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts)
          .then((res) => res.json())
@@ -122,7 +122,7 @@ const runAppCronJobs = () => {
       if (notif_interval) {
          const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
          if (cronTime) {
-            cron.schedule(cronTime, () => {
+            Cron(cronTime, () => {
                // console.log('### Sending Notification Email...');
                const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
                fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)

database/database.ts

@@ -6,7 +6,7 @@ import Keyword from './models/keyword';
 const connection = new Sequelize({
    dialect: 'sqlite',
    host: '0.0.0.0',
-   username: process.env.USER,
+   username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER,
    password: process.env.PASSWORD,
    database: 'sequelize',
    dialectModule: sqlite3,

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "serpbear",
-  "version": "0.2.1",
+  "version": "0.2.4",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "serpbear",
-      "version": "0.2.1",
+      "version": "0.2.4",
       "dependencies": {
         "@googleapis/searchconsole": "^1.0.0",
         "@testing-library/react": "^13.4.0",
@@ -17,6 +17,7 @@
         "cheerio": "^1.0.0-rc.12",
         "concurrently": "^7.6.0",
         "cookies": "^0.8.0",
+        "croner": "^5.3.5",
         "cryptr": "^6.0.3",
         "dayjs": "^1.11.5",
         "dotenv": "^16.0.3",
@@ -3707,6 +3708,14 @@
         "node": ">=10"
       }
     },
+    "node_modules/croner": {
+      "version": "5.3.5",
+      "resolved": "https://registry.npmjs.org/croner/-/croner-5.3.5.tgz",
+      "integrity": "sha512-VqaplJOVtaGuAxhsw2HM9GG0DLpVi3W9IsV7bKMAC12O7wMIOcZpCYHBw+xkFABzT3xp5MvUqTfbTewCgxgN+A==",
+      "engines": {
+        "node": ">=6.0"
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -15220,6 +15229,11 @@
         "yaml": "^1.10.0"
       }
     },
+    "croner": {
+      "version": "5.3.5",
+      "resolved": "https://registry.npmjs.org/croner/-/croner-5.3.5.tgz",
+      "integrity": "sha512-VqaplJOVtaGuAxhsw2HM9GG0DLpVi3W9IsV7bKMAC12O7wMIOcZpCYHBw+xkFABzT3xp5MvUqTfbTewCgxgN+A=="
+    },
     "cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "0.2.1",
+  "version": "0.2.4",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -25,6 +25,7 @@
     "cheerio": "^1.0.0-rc.12",
     "concurrently": "^7.6.0",
     "cookies": "^0.8.0",
+    "croner": "^5.3.5",
     "cryptr": "^6.0.3",
     "dayjs": "^1.11.5",
     "dotenv": "^16.0.3",

pages/api/keywords.ts

@@ -153,10 +153,13 @@ const updateKeywords = async (req: NextApiRequest, res: NextApiResponse<Keywords
       }
       if (tags) {
          const tagsKeywordIDs = Object.keys(tags);
+         const multipleKeywords = tagsKeywordIDs.length > 1;
          for (const keywordID of tagsKeywordIDs) {
-            const response = await Keyword.findOne({ where: { ID: keywordID } });
-            if (response) {
-               await response.update({ tags: JSON.stringify(tags[keywordID]) });
+            const selectedKeyword = await Keyword.findOne({ where: { ID: keywordID } });
+            const currentTags = selectedKeyword && selectedKeyword.tags ? JSON.parse(selectedKeyword.tags) : [];
+            const mergedTags = Array.from(new Set([...currentTags, ...tags[keywordID]]));
+            if (selectedKeyword) {
+               await selectedKeyword.update({ tags: JSON.stringify(multipleKeywords ? mergedTags : tags[keywordID]) });
             }
          }
          return res.status(200).json({ keywords });

pages/api/login.ts

@@ -18,10 +18,11 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
    if (!req.body.username || !req.body.password) {
       return res.status(401).json({ error: 'Username Password Missing' });
    }
+   const userName = process.env.USER_NAME ? process.env.USER_NAME : process.env.USER;
 
-   if (req.body.username === process.env.USER
+   if (req.body.username === userName
       && req.body.password === process.env.PASSWORD && process.env.SECRET) {
-      const token = jwt.sign({ user: process.env.USER }, process.env.SECRET);
+      const token = jwt.sign({ user: userName }, process.env.SECRET);
       const cookies = new Cookies(req, res);
       const expireDate = new Date();
       const sessDuration = process.env.SESSION_DURATION;
@@ -30,7 +31,7 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
       return res.status(200).json({ success: true, error: null });
    }
 
-   const error = req.body.username !== process.env.USER ? 'Incorrect Username' : 'Incorrect Password';
+   const error = req.body.username !== userName ? 'Incorrect Username' : 'Incorrect Password';
 
    return res.status(401).json({ success: false, error });
 };

scrapers/services/scrapingant.ts

@@ -2,6 +2,11 @@ const scrapingAnt:ScraperSettings = {
    id: 'scrapingant',
    name: 'ScrapingAnt',
    website: 'scrapingant.com',
+   headers: (keyword) => {
+      // eslint-disable-next-line max-len
+      const mobileAgent = 'Mozilla/5.0 (Linux; Android 10; SM-G996U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36';
+      return keyword && keyword.device === 'mobile' ? { 'Ant-User-Agent': mobileAgent } : {};
+   },
    scrapeURL: (keyword, settings, countryData) => {
       const scraperCountries = ['AE', 'BR', 'CN', 'DE', 'ES', 'FR', 'GB', 'HK', 'PL', 'IN', 'IT', 'IL', 'JP', 'NL', 'RU', 'SA', 'US', 'CZ'];
       const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';

styles/globals.css

@@ -84,8 +84,8 @@ body {
 
 .domKeywords_head--alpha_desc .domKeywords_head_keyword::after,
 .domKeywords_head--pos_desc .domKeywords_head_position::after,
-.domKeywords_head--imp_desc .domKeywords_head_imp::after,
-.domKeywords_head--visits_desc .domKeywords_head_visits::after,
+.domKeywords_head--imp_asc .domKeywords_head_imp::after,
+.domKeywords_head--visits_asc .domKeywords_head_visits::after,
 .domKeywords_head--ctr_desc .domKeywords_head_ctr::after {
    content: "↓";
    display: inline-block;
@@ -98,8 +98,8 @@ body {
 
 .domKeywords_head--alpha_asc .domKeywords_head_keyword::after,
 .domKeywords_head--pos_asc .domKeywords_head_position::after,
-.domKeywords_head--imp_asc .domKeywords_head_imp::after,
-.domKeywords_head--visits_asc .domKeywords_head_visits::after,
+.domKeywords_head--imp_desc .domKeywords_head_imp::after,
+.domKeywords_head--visits_desc .domKeywords_head_visits::after,
 .domKeywords_head--ctr_asc .domKeywords_head_ctr::after {
    content: "↑";
    display: inline-block;

utils/SCsortFilter.ts

@@ -9,16 +9,16 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
    const keywords = theKeywords.map((k) => ({ ...k, position: k.position === 0 ? 111 : k.position }));
    switch (sortBy) {
       case 'imp_asc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.impressions - a.impressions);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.impressions > b.impressions ? 1 : -1));
             break;
       case 'imp_desc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.impressions - b.impressions);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.impressions > a.impressions ? 1 : -1));
             break;
       case 'visits_asc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.clicks - a.clicks);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.clicks > b.clicks ? 1 : -1));
             break;
       case 'visits_desc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.clicks - b.clicks);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.clicks > a.clicks ? 1 : -1));
             break;
        case 'ctr_asc':
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.ctr - a.ctr);
@@ -27,17 +27,17 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.ctr - b.ctr);
             break;
       case 'pos_asc':
-            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position > a.position ? 1 : -1));
+            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position < a.position ? 1 : -1));
             sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
             break;
       case 'pos_desc':
-            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position > b.position ? 1 : -1));
+            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position < b.position ? 1 : -1));
             sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
             break;
-      case 'alpha_asc':
+      case 'alpha_desc':
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.keyword > a.keyword ? 1 : -1));
             break;
-      case 'alpha_desc':
+      case 'alpha_asc':
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.keyword > b.keyword ? 1 : -1));
          break;
       default:

utils/scraper.ts

@@ -49,7 +49,7 @@ export const getScraperClient = (keyword:KeywordType, settings:SettingsType, scr
       // Set Scraper Header
       const scrapeHeaders = scraper.headers ? scraper.headers(keyword, settings) : null;
       const scraperAPIURL = scraper.scrapeURL ? scraper.scrapeURL(keyword, settings, countries) : null;
-      if (scrapeHeaders) {
+      if (scrapeHeaders && Object.keys(scrapeHeaders).length > 0) {
          Object.keys(scrapeHeaders).forEach((headerItemKey:string) => {
             headers[headerItemKey] = scrapeHeaders[headerItemKey as keyof object];
          });
@@ -114,7 +114,7 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
       const scraperResult = scraperObj?.resultObjectKey && res[scraperObj.resultObjectKey] ? res[scraperObj.resultObjectKey] : '';
       const scrapeResult:string = (res.data || res.html || res.results || scraperResult || '');
       if (res && scrapeResult) {
-         const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult);
+         const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult, keyword.device);
          // await writeFile('result.txt', JSON.stringify(scrapeResult), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
          const serp = getSerp(keyword.domain, extracted);
          refreshedResults = { ID: keyword.ID, keyword: keyword.keyword, position: serp.postion, url: serp.url, result: extracted, error: false };
@@ -141,9 +141,10 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
 /**
  * Extracts the Google Search result as object array from the Google Search's HTML content
  * @param {string} content - scraped google search page html data.
+ * @param {string} device - The device of the keyword.
  * @returns {SearchResult[]}
  */
-export const extractScrapedResult = (content: string): SearchResult[] => {
+export const extractScrapedResult = (content: string, device: string): SearchResult[] => {
    const extractedResult = [];
 
    const $ = cheerio.load(content);
@@ -163,6 +164,24 @@ export const extractScrapedResult = (content: string): SearchResult[] => {
       }
   }
 
+  // Mobile Scraper
+  if (extractedResult.length === 0 && device === 'mobile') {
+      const items = $('body').find('#rso > div');
+      for (let i = 0; i < items.length; i += 1) {
+         const item = $(items[i]);
+         const linkDom = item.find('a[role="presentation"]');
+         if (linkDom) {
+            const url = linkDom.attr('href');
+            const titleDom = linkDom.find('[role="link"]');
+            const title = titleDom ? titleDom.text() : '';
+            if (title && url) {
+               lastPosition += 1;
+               extractedResult.push({ title, url, position: lastPosition });
+            }
+         }
+      }
+   }
+
   return extractedResult;
 };
 
@@ -193,7 +212,7 @@ export const retryScrape = async (keywordID: number) : Promise<void> => {
 
    const filePath = `${process.cwd()}/data/failed_queue.json`;
    const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
-   currentQueue = JSON.parse(currentQueueRaw);
+   currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
 
    if (!currentQueue.includes(keywordID)) {
       currentQueue.push(keywordID);
@@ -213,7 +232,7 @@ export const removeFromRetryQueue = async (keywordID: number) : Promise<void> =>
 
    const filePath = `${process.cwd()}/data/failed_queue.json`;
    const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
-   currentQueue = JSON.parse(currentQueueRaw);
+   currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
    currentQueue = currentQueue.filter((item) => item !== keywordID);
 
    await writeFile(filePath, JSON.stringify(currentQueue), { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });

utils/searchConsole.ts

@@ -54,7 +54,7 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
                date: row.keys[0],
                clicks: row.clicks,
                impressions: row.impressions,
-               ctr: row.ctr,
+               ctr: row.ctr * 100,
                position: row.position,
             });
          });
@@ -101,7 +101,7 @@ export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainNam
    const page = SCItem.keys[3] ? SCItem.keys[3].replace('https://', '').replace('http://', '').replace('www', '').replace(domainName, '') : '';
    const uid = `${country.toLowerCase()}:${device}:${keyword.replaceAll(' ', '_')}`;
 
-   return { keyword, uid, device, country, clicks, impressions, ctr, position, page };
+   return { keyword, uid, device, country, clicks, impressions, ctr: ctr * 100, position, page };
 };
 
 export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainDataType) : KeywordType => {

utils/sortFilter.ts

@@ -28,7 +28,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
       case 'alpha_desc':
             sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.keyword > b.keyword ? 1 : -1));
          break;
-      case 'imp_asc':
+      case 'imp_desc':
             if (scDataType) {
                   sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
                   const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
@@ -37,7 +37,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
                });
             }
             break;
-      case 'imp_desc':
+      case 'imp_asc':
             if (scDataType) {
                   sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
                   const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
@@ -46,21 +46,21 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
                });
             }
          break;
-      case 'visits_asc':
-            if (scDataType) {
-                  sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
-                  const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  return aImpressionData > bImpressionData ? 1 : -1;
-               });
-            }
-            break;
       case 'visits_desc':
             if (scDataType) {
                   sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
-                  const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  return bImpressionData > aImpressionData ? 1 : -1;
+                  const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  return aVisitsData > bVisitsData ? 1 : -1;
+               });
+            }
+            break;
+      case 'visits_asc':
+            if (scDataType) {
+                  sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
+                  const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  return bVisitsData > aVisitsData ? 1 : -1;
                });
             }
             break;