chore(release): 0.2.6

fixes #87

CHANGELOG.md

@@ -2,6 +2,71 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.2.6](https://github.com/towfiqi/serpbear/compare/v0.2.5...v0.2.6) (2023-03-29)
+
+
+### Features
+
+* Add option to Delay Between scrapes. ([0a83924](https://github.com/towfiqi/serpbear/commit/0a83924ffe2243c52849c167c6c15d9688ff1dc7)), closes [#87](https://github.com/towfiqi/serpbear/issues/87)
+* Integrates Space Serp. ([0538a8c](https://github.com/towfiqi/serpbear/commit/0538a8c01601d2f6365848580591a248528e67c7))
+
+
+### Bug Fixes
+
+* **components:** fix typo "Goolge" -> "Google" ([dce7c41](https://github.com/towfiqi/serpbear/commit/dce7c412e813fc845973f36ad1c9fa91df4a6611))
+* Fixes first Keryword Error cut off issue. ([d950515](https://github.com/towfiqi/serpbear/commit/d9505158c439a924a1c86eb8243faf2a15bed43e))
+* Fixes lags when tracking thousands of keywords ([9757fde](https://github.com/towfiqi/serpbear/commit/9757fde02ec83405546733381104c54ed6510681)), closes [#88](https://github.com/towfiqi/serpbear/issues/88)
+
+### [0.2.5](https://github.com/towfiqi/serpbear/compare/v0.2.4...v0.2.5) (2023-03-07)
+
+
+### Features
+
+* Adds current App version Number in Footer. ([b83df5f](https://github.com/towfiqi/serpbear/commit/b83df5f3dbd64db657d31f0526438e7165e1b475))
+* Adds Keyword Scraping Interval Settings. ([3b6d034](https://github.com/towfiqi/serpbear/commit/3b6d034d6f7da0b4259070220fffff44184dd680)), closes [#81](https://github.com/towfiqi/serpbear/issues/81) [#76](https://github.com/towfiqi/serpbear/issues/76)
+
+
+### Bug Fixes
+
+* Fixes Broken Image thumbnail loading issue. ([5dd366b](https://github.com/towfiqi/serpbear/commit/5dd366b91e2a94e658bf5250a8a0fa64c09e1c11))
+* Settings Update Toast was not showing up. ([b9d58a7](https://github.com/towfiqi/serpbear/commit/b9d58a721df12f3f34220a3ae5da6897e23c83ec))
+
+### [0.2.4](https://github.com/towfiqi/serpbear/compare/v0.2.3...v0.2.4) (2023-02-15)
+
+
+### Features
+
+* Keyword ranking pages can now be clicked. ([c5af94a](https://github.com/towfiqi/serpbear/commit/c5af94a1469713ed4092253d26953ee0ed28c25d))
+
+
+### Bug Fixes
+
+* Fixes broken Login on windows ([c406588](https://github.com/towfiqi/serpbear/commit/c406588953035e4177a64011c13eb0e3aedffe89))
+* Fixes Node Cron memory leak issue. ([3c48d13](https://github.com/towfiqi/serpbear/commit/3c48d130b6f229a4ac27ec43ef1ea3a6640cecf6))
+
+### [0.2.3](https://github.com/towfiqi/serpbear/compare/v0.2.2...v0.2.3) (2023-01-12)
+
+
+### Features
+
+* Ability to tag multiple keywords at once ([9e9dad7](https://github.com/towfiqi/serpbear/commit/9e9dad7631691b2a836fdd4c522b1f933b17e285)), closes [#54](https://github.com/towfiqi/serpbear/issues/54)
+* Set USERNAME as well as USER variable ([b50733d](https://github.com/towfiqi/serpbear/commit/b50733defc2c06e0f92ca3e88fd1f74684eee9c0))
+
+
+### Bug Fixes
+
+* Fixes Position and View Sort. ([8139e39](https://github.com/towfiqi/serpbear/commit/8139e399c13ab8be767facef9a19c67dec06ed64)), closes [#46](https://github.com/towfiqi/serpbear/issues/46)
+* Fixes wrong CTR value for Search Console Data ([cb24696](https://github.com/towfiqi/serpbear/commit/cb24696a1f47b02a11c68cd1c673ea8b1bacd144)), closes [#48](https://github.com/towfiqi/serpbear/issues/48)
+* Mobile Keyword Scraping not working. ([a1108d2](https://github.com/towfiqi/serpbear/commit/a1108d240ea38ab0886ef3722b0c937ec5a45591)), closes [#58](https://github.com/towfiqi/serpbear/issues/58)
+* ScrapingAnt Mobile Keyword Scrape not working ([acc0b39](https://github.com/towfiqi/serpbear/commit/acc0b39d80d4f9371967a0d425ed205c5d866eea))
+
+### [0.2.2](https://github.com/towfiqi/serpbear/compare/v0.2.1...v0.2.2) (2022-12-25)
+
+
+### Bug Fixes
+
+* Fixes bug that prevents Saving API settings ([123ad81](https://github.com/towfiqi/serpbear/commit/123ad81dae10aa28848148d0f3da5cf1f7de7c57)), closes [#45](https://github.com/towfiqi/serpbear/issues/45)
+
 ### [0.2.1](https://github.com/towfiqi/serpbear/compare/v0.2.0...v0.2.1) (2022-12-24)
 
 ## [0.2.0](https://github.com/towfiqi/serpbear/compare/v0.1.7...v0.2.0) (2022-12-21)

Dockerfile

@@ -29,7 +29,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/cron.js ./
 COPY --from=builder --chown=nextjs:nodejs /app/email ./email
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv node-cron @googleapis/searchconsole
+RUN npm i cryptr dotenv croner @googleapis/searchconsole
 RUN npm i -g concurrently
 
 USER nextjs

README.md

@@ -40,6 +40,7 @@ The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SerpA
 | whatsmyserp.com | $49/mo| 30,000/mo| No |
 | serply.io | $49/mo | 5000/mo | Yes |
 | serpapi.com | From $50/mo** | From 5,000/mo** | Yes |
+| spaceserp.com | $59/lifetime | 15,000/mo | Yes |
 
 (*) Free upto a limit. If you are using ScrapingAnt you can lookup 10,000 times per month for free.
 (**) Free up to 100 per month. Paid from 5,000 to 10,000,000+ per month.

components/domains/DomainItem.tsx

@@ -9,10 +9,11 @@ import Icon from '../common/Icon';
 type DomainItemProps = {
    domain: DomainType,
    selected: boolean,
-   isConsoleIntegrated: boolean
+   isConsoleIntegrated: boolean,
+   thumb: string,
 }
 
-const DomainItem = ({ domain, selected, isConsoleIntegrated = false }: DomainItemProps) => {
+const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: DomainItemProps) => {
    const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain;
    // const router = useRouter();
    return (
@@ -21,10 +22,10 @@ const DomainItem = ({ domain, selected, isConsoleIntegrated = false }: DomainIte
          <a className='flex flex-col lg:flex-row'>
             <div className={`flex-1 p-6 flex ${!isConsoleIntegrated ? 'basis-1/3' : ''}`}>
                <div className="domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded border border-gray-200 overflow-hidden">
-                  <img src={`https://image.thum.io/get/maxAge/96/width/200/https://${domain.domain}`} alt={domain.domain} />
+                  {thumb && <img src={thumb} alt={domain.domain} />}
                </div>
                <div className="domain_details flex-1">
-                  <h3 className='font-semibold text-base mb-2 capitalize'>{domain.domain}</h3>
+                  <h3 className='font-semibold text-base mb-2'>{domain.domain}</h3>
                  {keywordsUpdated && (
                   <span className=' text-gray-600 text-xs'>
                      Updated <TimeAgo title={dayjs(keywordsUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={keywordsUpdated} />

components/insight/Insight.tsx

@@ -117,7 +117,7 @@ const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SC
                      )}
                      {!isConsoleIntegrated && (
                         <p className=' p-9 pt-[10%] text-center text-gray-500'>
-                        Goolge Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        Google Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
                         </p>
                      )}
                   </div>

components/keywords/AddTags.tsx

@@ -0,0 +1,60 @@
+import { useState } from 'react';
+import { useUpdateKeywordTags } from '../../services/keywords';
+import Icon from '../common/Icon';
+import Modal from '../common/Modal';
+
+type AddTagsProps = {
+   keywords: KeywordType[],
+   closeModal: Function
+}
+
+const AddTags = ({ keywords = [], closeModal }: AddTagsProps) => {
+   const [tagInput, setTagInput] = useState('');
+   const [inputError, setInputError] = useState('');
+   const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
+
+   const addTag = () => {
+      if (keywords.length === 0) { return; }
+      if (!tagInput) {
+         setInputError('Please Insert a Tag!');
+         setTimeout(() => { setInputError(''); }, 3000);
+         return;
+      }
+
+      const tagsArray = tagInput.split(',').map((t) => t.trim());
+      const tagsPayload:any = {};
+      keywords.forEach((keyword:KeywordType) => {
+         tagsPayload[keyword.ID] = [...keyword.tags, ...tagsArray];
+      });
+      updateMutate({ tags: tagsPayload });
+   };
+
+   return (
+      <Modal closeModal={() => { closeModal(false); }} title={`Add New Tags to ${keywords.length} Selected Keyword`}>
+         <div className="relative">
+            {inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
+            <span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
+            <input
+               className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
+               placeholder='Insert Tags. eg: tag1, tag2'
+               value={tagInput}
+               onChange={(e) => setTagInput(e.target.value)}
+               onKeyDown={(e) => {
+                  if (e.code === 'Enter') {
+                    e.preventDefault();
+                    addTag();
+                  }
+               }}
+            />
+            <button
+            className=" absolute right-2 top-2 cursor-pointer rounded p-2 px-4 bg-indigo-600 text-white font-semibold text-sm"
+            onClick={addTag}>
+               Apply
+            </button>
+         </div>
+      </Modal>
+
+   );
+};
+
+export default AddTags;

components/keywords/Keyword.tsx

@@ -9,6 +9,7 @@ import { generateTheChartData } from '../common/generateChartData';
 type KeywordProps = {
    keywordData: KeywordType,
    selected: boolean,
+   index: number,
    refreshkeyword: Function,
    favoriteKeyword: Function,
    removeKeyword: Function,
@@ -18,6 +19,7 @@ type KeywordProps = {
    lastItem?:boolean,
    showSCData: boolean,
    scDataType: string,
+   style: Object
 }
 
 const Keyword = (props: KeywordProps) => {
@@ -32,6 +34,8 @@ const Keyword = (props: KeywordProps) => {
       manageTags,
       lastItem,
       showSCData = true,
+      style,
+      index,
       scDataType = 'threeDays',
    } = props;
    const {
@@ -78,6 +82,7 @@ const Keyword = (props: KeywordProps) => {
    return (
       <div
       key={keyword}
+      style={style}
       className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
       lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
          <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold cursor-pointer'>
@@ -115,7 +120,10 @@ const Keyword = (props: KeywordProps) => {
          <div
          className={`keyword_url inline-block mt-4 mr-5 ml-5 lg:flex-1 text-gray-400 lg:m-0 max-w-[70px] 
          overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5`}>
-            <span className='mr-3 lg:hidden'><Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}</div>
+            <a href={url} target="_blank" rel="noreferrer"><span className='mr-3 lg:hidden'>
+               <Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}
+            </a>
+         </div>
          <div
          className='inline-block mt-[4] top-[-5px] relative lg:flex-1 lg:m-0'>
             <span className='mr-2 lg:hidden'><Icon type="clock" size={14} color="#999" /></span>
@@ -169,7 +177,8 @@ const Keyword = (props: KeywordProps) => {
             )}
          </div>
          {lastUpdateError && lastUpdateError.date && showPositionError && (
-            <div className=' absolute mt-[-70px] p-2 bg-white z-30 border border-red-200 rounded w-[220px] left-4 shadow-sm text-xs lg:bottom-12'>
+            <div className={`absolute p-2 bg-white z-30 border border-red-200 rounded w-[220px] left-4 shadow-sm text-xs 
+            ${index > 2 ? 'lg:bottom-12 mt-[-70px]' : ' top-12'}`}>
                Error Updating Keyword position (Tried <TimeAgo
                                                          title={dayjs(lastUpdateError.date).format('DD-MMM-YYYY, hh:mm:ss A')}
                                                          date={lastUpdateError.date} />)

components/keywords/KeywordDetails.tsx

@@ -85,7 +85,7 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
                   <h3 className=' text-lg font-bold'>
                      <span title={countries[keyword.country][0]}
                      className={`fflag fflag-${keyword.country} w-[18px] h-[12px] mr-2`} /> {keyword.keyword}
-                     <span className='py-1 px-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
+                     <span className='py-1 px-2 ml-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
                   </h3>
                   <button
                   className='absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'

components/keywords/KeywordFilter.tsx

@@ -82,10 +82,10 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       { value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
    ];
    if (integratedConsole) {
-      sortOptionChoices.push({ value: 'imp_asc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
-      sortOptionChoices.push({ value: 'imp_desc', label: 'Least Viewed' });
-      sortOptionChoices.push({ value: 'visits_asc', label: 'Most Visited' });
-      sortOptionChoices.push({ value: 'visits_desc', label: 'Least Visited' });
+      sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
+      sortOptionChoices.push({ value: 'imp_asc', label: 'Least Viewed' });
+      sortOptionChoices.push({ value: 'visits_desc', label: 'Most Visited' });
+      sortOptionChoices.push({ value: 'visits_asc', label: 'Least Visited' });
    }
    if (isConsole) {
       sortOptionChoices.splice(2, 2);

components/keywords/KeywordTagManager.tsx

@@ -2,6 +2,7 @@ import { useState } from 'react';
 import { useUpdateKeywordTags } from '../../services/keywords';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
+import AddTags from './AddTags';
 
 type keywordTagManagerProps = {
    keyword: KeywordType|undefined,
@@ -10,9 +11,8 @@ type keywordTagManagerProps = {
 }
 
 const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
-   const [tagInput, setTagInput] = useState('');
-   const [inputError, setInputError] = useState('');
-   const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
+   const [showAddTag, setShowAddTag] = useState<boolean>(false);
+   const { mutate: updateMutate } = useUpdateKeywordTags(() => { });
 
    const removeTag = (tag:String) => {
       if (!keyword) { return; }
@@ -20,24 +20,6 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
       updateMutate({ tags: { [keyword.ID]: newTags } });
    };
 
-   const addTag = () => {
-      if (!keyword) { return; }
-      if (!tagInput) {
-         setInputError('Please Insert a Tag!');
-         setTimeout(() => { setInputError(''); }, 3000);
-         return;
-      }
-      if (keyword.tags.includes(tagInput)) {
-         setInputError('Tag Exist!');
-         setTimeout(() => { setInputError(''); }, 3000);
-         return;
-      }
-
-      console.log('New Tag: ', tagInput);
-      const newTags = [...keyword.tags, tagInput.trim()];
-      updateMutate({ tags: { [keyword.ID]: newTags } });
-   };
-
    return (
       <Modal closeModal={() => { closeModal(false); }} title={`Tags for Keyword "${keyword && keyword.keyword}"`}>
          <div className="text-sm my-8 ">
@@ -53,31 +35,27 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
                                  </button>
                            </li>;
                   })}
+                  <li className='inline-block py-1 px-1'>
+                     <button
+                     title='Add New Tag'
+                     className="cursor-pointer rounded p-1 px-3 bg-indigo-600 text-white font-semibold text-sm"
+                     onClick={() => setShowAddTag(true)}>+</button>
+                  </li>
                </ul>
             )}
             {keyword && keyword.tags.length === 0 && (
-               <div className="text-center w-full text-gray-500">No Tags Added to this Keyword.</div>
+               <div className="text-center w-full text-gray-500">
+                  No Tags Added to this Keyword. <button className=' text-indigo-600' onClick={() => setShowAddTag(true)}>+ Add Tag</button>
+               </div>
             )}
          </div>
-         <div className="relative">
-            {inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
-            <span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
-            <input
-               className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
-               placeholder='Insert Tags'
-               value={tagInput}
-               onChange={(e) => setTagInput(e.target.value)}
-               onKeyDown={(e) => {
-                  if (e.code === 'Enter') {
-                    e.preventDefault();
-                    addTag();
-                  }
-               }}
-            />
-            <button className=" absolute right-2 top-2 cursor-pointer rounded p-1 px-4 bg-blue-600 text-white font-bold" onClick={addTag}>+</button>
-         </div>
+         {showAddTag && keyword && (
+            <AddTags
+               keywords={[keyword]}
+               closeModal={() => setShowAddTag(false)}
+               />
+         )}
       </Modal>
-
    );
 };
 

components/keywords/KeywordsTable.tsx

@@ -1,6 +1,7 @@
-import React, { useState, useMemo } from 'react';
+import React, { useState, useMemo, useEffect } from 'react';
 import { Toaster } from 'react-hot-toast';
 import { CSSTransition } from 'react-transition-group';
+import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
 import AddKeywords from './AddKeywords';
 import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/sortFilter';
 import Icon from '../common/Icon';
@@ -10,6 +11,7 @@ import KeywordFilters from './KeywordFilter';
 import Modal from '../common/Modal';
 import { useDeleteKeywords, useFavKeywords, useRefreshKeywords } from '../../services/keywords';
 import KeywordTagManager from './KeywordTagManager';
+import AddTags from './AddTags';
 
 type KeywordsTableProps = {
    domain: DomainType | null,
@@ -28,6 +30,9 @@ const KeywordsTable = (props: KeywordsTableProps) => {
    const [showKeyDetails, setShowKeyDetails] = useState<KeywordType|null>(null);
    const [showRemoveModal, setShowRemoveModal] = useState<boolean>(false);
    const [showTagManager, setShowTagManager] = useState<null|number>(null);
+   const [showAddTags, setShowAddTags] = useState<boolean>(false);
+   const [isMobile, setIsMobile] = useState<boolean>(false);
+   const [SCListHeight, setSCListHeight] = useState(500);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
    const [sortBy, setSortBy] = useState<string>('date_asc');
    const [scDataType, setScDataType] = useState<string>('threeDays');
@@ -45,6 +50,16 @@ const KeywordsTable = (props: KeywordsTableProps) => {
       avgThirtyDays: 'Last Thirty Days Avg',
    };
 
+   useEffect(() => {
+      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
+      const resizeList = () => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400));
+      resizeList();
+      window.addEventListener('resize', resizeList);
+      return () => {
+          window.removeEventListener('resize', resizeList);
+      };
+   }, [isMobile]);
+
    const processedKeywords: {[key:string] : KeywordType[]} = useMemo(() => {
       const procKeywords = keywords.filter((x) => x.device === device);
       const filteredKeywords = filterKeywords(procKeywords, filterParams);
@@ -65,6 +80,27 @@ const KeywordsTable = (props: KeywordsTableProps) => {
       }
       setSelectedKeywords(updatedSelectd);
    };
+   const Row = ({ data, index, style }:ListChildComponentProps) => {
+      const keyword = data[index];
+      return (
+         <Keyword
+         key={keyword.ID}
+         style={style}
+         index={index}
+         selected={selectedKeywords.includes(keyword.ID)}
+         selectKeyword={selectKeyword}
+         keywordData={keyword}
+         refreshkeyword={() => refreshMutate({ ids: [keyword.ID] })}
+         favoriteKeyword={favoriteMutate}
+         manageTags={() => setShowTagManager(keyword.ID)}
+         removeKeyword={() => { setSelectedKeywords([keyword.ID]); setShowRemoveModal(true); }}
+         showKeywordDetails={() => setShowKeyDetails(keyword)}
+         lastItem={index === (processedKeywords[device].length - 1)}
+         showSCData={showSCData}
+         scDataType={scDataType}
+         />
+      );
+   };
 
    const selectedAllItems = selectedKeywords.length === processedKeywords[device].length;
 
@@ -79,7 +115,7 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                         className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
                         onClick={() => { refreshMutate({ ids: selectedKeywords }); setSelectedKeywords([]); }}
                         >
-                           <span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keyword
+                           <span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keywords
                         </a>
                      </li>
                      <li className='inline-block mr-4'>
@@ -87,7 +123,14 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                         className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
                         onClick={() => setShowRemoveModal(true)}
                         >
-                           <span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keyword</a>
+                           <span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keywords</a>
+                     </li>
+                     <li className='inline-block mr-4'>
+                        <a
+                        className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
+                        onClick={() => setShowAddTags(true)}
+                        >
+                           <span className=' bg-green-100 text-green-500  px-1 rounded'><Icon type="tags" size={14} /></span> Tag Keywords</a>
                      </li>
                   </ul>
                </div>
@@ -161,21 +204,19 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                      )}
                   </div>
                   <div className='domKeywords_keywords border-gray-200 min-h-[55vh] relative'>
-                     {processedKeywords[device] && processedKeywords[device].length > 0
-                        && processedKeywords[device].map((keyword, index) => <Keyword
-                                                      key={keyword.ID}
-                                                      selected={selectedKeywords.includes(keyword.ID)}
-                                                      selectKeyword={selectKeyword}
-                                                      keywordData={keyword}
-                                                      refreshkeyword={() => refreshMutate({ ids: [keyword.ID] })}
-                                                      favoriteKeyword={favoriteMutate}
-                                                      manageTags={() => setShowTagManager(keyword.ID)}
-                                                      removeKeyword={() => { setSelectedKeywords([keyword.ID]); setShowRemoveModal(true); }}
-                                                      showKeywordDetails={() => setShowKeyDetails(keyword)}
-                                                      lastItem={index === (processedKeywords[device].length - 1)}
-                                                      showSCData={showSCData}
-                                                      scDataType={scDataType}
-                                                      />)}
+                     {processedKeywords[device] && processedKeywords[device].length > 0 && (
+                        <List
+                        innerElementType="div"
+                        itemData={processedKeywords[device]}
+                        itemCount={processedKeywords[device].length}
+                        itemSize={isMobile ? 146 : 57}
+                        height={SCListHeight}
+                        width={'100%'}
+                        className={'styled-scrollbar'}
+                        >
+                           {Row}
+                        </List>
+                     )}
                      {!isLoading && processedKeywords[device].length === 0 && (
                         <p className=' p-9 pt-[10%] text-center text-gray-500'>No Keywords Added for this Device Type.</p>
                      )}
@@ -222,6 +263,12 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                closeModal={() => setShowTagManager(null)}
                />
          )}
+         {showAddTags && (
+            <AddTags
+               keywords={keywords.filter((k) => selectedKeywords.includes(k.ID))}
+               closeModal={() => setShowAddTags(false)}
+               />
+         )}
          <Toaster position='bottom-center' containerClassName="react_toaster" />
       </div>
    );

components/keywords/SCKeywordsTable.tsx

@@ -26,7 +26,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
    const [device, setDevice] = useState<string>('desktop');
    const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
-   const [sortBy, setSortBy] = useState<string>('imp_asc');
+   const [sortBy, setSortBy] = useState<string>('imp_desc');
    const [isMobile, setIsMobile] = useState<boolean>(false);
    const [SCListHeight, setSCListHeight] = useState(500);
    const { keywordsData } = useFetchKeywords(router);
@@ -214,7 +214,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
                      )}
                      {!isConsoleIntegrated && (
                         <p className=' p-9 pt-[10%] text-center text-gray-500'>
-                        Goolge Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        Google Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
                         </p>
                      )}
                   </div>

components/settings/Settings.tsx

@@ -1,4 +1,5 @@
 import React, { useEffect, useState } from 'react';
+import { Toaster } from 'react-hot-toast';
 // import { useQuery } from 'react-query';
 import useUpdateSettings, { useFetchSettings } from '../../services/settings';
 import Icon from '../common/Icon';
@@ -16,6 +17,7 @@ type SettingsError = {
 
 const defaultSettings = {
    scraper_type: 'none',
+   scrape_delay: 'none',
    notification_interval: 'daily',
    notification_email: '',
    smtp_server: '',
@@ -63,21 +65,20 @@ const Settings = ({ closeSettings }:SettingsProps) => {
 
    const performUpdate = () => {
       let error: null|SettingsError = null;
-      if (settings.notification_interval !== 'never') {
+      const { notification_interval, notification_email, notification_email_from, scraper_type, smtp_port, smtp_server, scaping_api } = settings;
+      if (notification_interval !== 'never') {
          if (!settings.notification_email) {
             error = { type: 'no_email', msg: 'Insert a Valid Email address' };
          }
-         if (settings.notification_email
-            && (!settings.smtp_port || !settings.smtp_server
-               || !settings.notification_email_from)) {
+         if (notification_email && (!smtp_port || !smtp_server || !notification_email_from)) {
                let type = 'no_smtp_from';
-               if (!settings.smtp_port) { type = 'no_smtp_port'; }
-               if (!settings.smtp_server) { type = 'no_smtp_server'; }
+               if (!smtp_port) { type = 'no_smtp_port'; }
+               if (!smtp_server) { type = 'no_smtp_server'; }
                error = { type, msg: 'Insert SMTP Server details that will be used to send the emails.' };
          }
       }
 
-      if (settings?.scraper_type !== 'proxy' && settings?.scraper_type !== 'none') {
+      if (scraper_type !== 'proxy' && scraper_type !== 'none' && !scaping_api) {
          error = { type: 'no_api_key', msg: 'Insert a Valid API Key or Token for the Scraper Service.' };
       }
 
@@ -98,6 +99,25 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       { label: 'Monthly', value: 'monthly' },
       { label: 'Never', value: 'never' },
    ];
+   const scrapingOptions: SelectionOption[] = [
+      { label: 'Daily', value: 'daily' },
+      { label: 'Every Other Day', value: 'other_day' },
+      { label: 'Weekly', value: 'weekly' },
+      { label: 'Monthly', value: 'monthly' },
+      { label: 'Never', value: 'never' },
+   ];
+   const delayOptions: SelectionOption[] = [
+      { label: 'No Delay', value: '0' },
+      { label: '5 Seconds', value: '5000' },
+      { label: '10 Seconds', value: '10000' },
+      { label: '30 Seconds', value: '30000' },
+      { label: '1 Minutes', value: '60000' },
+      { label: '2 Minutes', value: '120000' },
+      { label: '5 Minutes', value: '300000' },
+      { label: '10 Minutes', value: '600000' },
+      { label: '15 Minutes', value: '900000' },
+      { label: '30 Minutes', value: '1800000' },
+   ];
    const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : [];
    const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers];
 
@@ -144,7 +164,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                            minWidth={270}
                            />
                         </div>
-                        {['scrapingant', 'scrapingrobot', 'serply', 'serpapi'].includes(settings.scraper_type) && (
+                        {['scrapingant', 'scrapingrobot', 'serply', 'serpapi', 'spaceSerp'].includes(settings.scraper_type) && (
                            <div className="settings__section__input mr-3">
                               <label className={labelStyle}>Scraper API Key or Token</label>
                               <input
@@ -170,6 +190,36 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                               />
                            </div>
                         )}
+                        {settings.scraper_type !== 'none' && (
+                           <div className="settings__section__input mb-5">
+                              <label className={labelStyle}>Scraping Frequency</label>
+                              <SelectField
+                                 multiple={false}
+                                 selected={[settings?.scrape_interval || 'daily']}
+                                 options={scrapingOptions}
+                                 defaultLabel={'Notification Settings'}
+                                 updateField={(updated:string[]) => updated[0] && updateSettings('scrape_interval', updated[0])}
+                                 rounded='rounded'
+                                 maxHeight={48}
+                                 minWidth={270}
+                              />
+                              <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
+                           </div>
+                        )}
+                           <div className="settings__section__input mb-5">
+                              <label className={labelStyle}>Delay Between Each keyword Scrape</label>
+                              <SelectField
+                                 multiple={false}
+                                 selected={[settings?.scrape_delay || '0']}
+                                 options={delayOptions}
+                                 defaultLabel={'Delay Settings'}
+                                 updateField={(updated:string[]) => updated[0] && updateSettings('scrape_delay', updated[0])}
+                                 rounded='rounded'
+                                 maxHeight={48}
+                                 minWidth={270}
+                              />
+                              <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
+                           </div>
                      </div>
                   </div>
                )}
@@ -271,6 +321,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                   </button>
                </div>
             </div>
+            <Toaster position='bottom-center' containerClassName="react_toaster" />
        </div>
    );
 };

cron.js

@@ -1,7 +1,7 @@
 const Cryptr = require('cryptr');
 const { promises } = require('fs');
 const { readFile } = require('fs');
-const cron = require('node-cron');
+const Cron = require('croner');
 require('dotenv').config({ path: './.env.local' });
 
 const getAppSettings = async () => {
@@ -49,6 +49,9 @@ const generateCronTime = (interval) => {
    if (interval === 'daily') {
       cronTime = '0 0 0 * * *';
    }
+   if (interval === 'other_day') {
+      cronTime = '0 0 2-30/2 * *';
+   }
    if (interval === 'daily_morning') {
       cronTime = '0 0 3 * * *';
    }
@@ -63,23 +66,47 @@ const generateCronTime = (interval) => {
 };
 
 const runAppCronJobs = () => {
-   // RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
-   const scrapeCronTime = generateCronTime('daily');
-   cron.schedule(scrapeCronTime, () => {
-      // console.log('### Running Keyword Position Cron Job!');
-      const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
-      fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
-      .then((res) => res.json())
-      .then((data) => console.log(data))
-      .catch((err) => {
-         console.log('ERROR Making Daily Scraper Cron Request..');
-         console.log(err);
-      });
-   }, { scheduled: true });
+   getAppSettings().then((settings) => {
+      // RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
+      const scrape_interval = settings.scrape_interval || 'daily';
+      if (scrape_interval !== 'never') {
+         const scrapeCronTime = generateCronTime(scrape_interval);
+         Cron(scrapeCronTime, () => {
+            // console.log('### Running Keyword Position Cron Job!');
+            const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
+            fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
+            .then((res) => res.json())
+            // .then((data) =>{ console.log(data)})
+            .catch((err) => {
+               console.log('ERROR Making SERP Scraper Cron Request..');
+               console.log(err);
+            });
+         }, { scheduled: true });
+      }
+
+      // RUN Email Notification CRON
+      const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
+      if (notif_interval) {
+         const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
+         if (cronTime) {
+            Cron(cronTime, () => {
+               // console.log('### Sending Notification Email...');
+               const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
+               fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
+               .then((res) => res.json())
+               .then((data) => console.log(data))
+               .catch((err) => {
+                  console.log('ERROR Making Cron Email Notification Request..');
+                  console.log(err);
+               });
+            }, { scheduled: true });
+         }
+      }
+   });
 
    // Run Failed scraping CRON (Every Hour)
    const failedCronTime = generateCronTime('hourly');
-   cron.schedule(failedCronTime, () => {
+   Cron(failedCronTime, () => {
       // console.log('### Retrying Failed Scrapes...');
 
       readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
@@ -104,7 +131,7 @@ const runAppCronJobs = () => {
    // Run Google Search Console Scraper Daily
    if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
       const searchConsoleCRONTime = generateCronTime('daily');
-      cron.schedule(searchConsoleCRONTime, () => {
+      Cron(searchConsoleCRONTime, () => {
          const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
          fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts)
          .then((res) => res.json())
@@ -115,27 +142,6 @@ const runAppCronJobs = () => {
          });
       }, { scheduled: true });
    }
-
-   // RUN Email Notification CRON
-   getAppSettings().then((settings) => {
-      const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
-      if (notif_interval) {
-         const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
-         if (cronTime) {
-            cron.schedule(cronTime, () => {
-               // console.log('### Sending Notification Email...');
-               const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
-               fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
-               .then((res) => res.json())
-               .then((data) => console.log(data))
-               .catch((err) => {
-                  console.log('ERROR Making Cron Email Notification Request..');
-                  console.log(err);
-               });
-            }, { scheduled: true });
-         }
-      }
-   });
 };
 
 runAppCronJobs();

database/database.ts

@@ -6,7 +6,7 @@ import Keyword from './models/keyword';
 const connection = new Sequelize({
    dialect: 'sqlite',
    host: '0.0.0.0',
-   username: process.env.USER,
+   username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER,
    password: process.env.PASSWORD,
    database: 'sequelize',
    dialectModule: sqlite3,

next.config.js

@@ -1,8 +1,13 @@
 /** @type {import('next').NextConfig} */
+const { version } = require('./package.json');
+
 const nextConfig = {
   reactStrictMode: true,
   swcMinify: false,
   output: 'standalone',
+  publicRuntimeConfig: {
+   version,
+ },
 };
 
 module.exports = nextConfig;

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "serpbear",
-  "version": "0.2.1",
+  "version": "0.2.6",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "serpbear",
-      "version": "0.2.1",
+      "version": "0.2.6",
       "dependencies": {
         "@googleapis/searchconsole": "^1.0.0",
         "@testing-library/react": "^13.4.0",
@@ -17,6 +17,7 @@
         "cheerio": "^1.0.0-rc.12",
         "concurrently": "^7.6.0",
         "cookies": "^0.8.0",
+        "croner": "^5.3.5",
         "cryptr": "^6.0.3",
         "dayjs": "^1.11.5",
         "dotenv": "^16.0.3",
@@ -3707,6 +3708,14 @@
         "node": ">=10"
       }
     },
+    "node_modules/croner": {
+      "version": "5.3.5",
+      "resolved": "https://registry.npmjs.org/croner/-/croner-5.3.5.tgz",
+      "integrity": "sha512-VqaplJOVtaGuAxhsw2HM9GG0DLpVi3W9IsV7bKMAC12O7wMIOcZpCYHBw+xkFABzT3xp5MvUqTfbTewCgxgN+A==",
+      "engines": {
+        "node": ">=6.0"
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -15220,6 +15229,11 @@
         "yaml": "^1.10.0"
       }
     },
+    "croner": {
+      "version": "5.3.5",
+      "resolved": "https://registry.npmjs.org/croner/-/croner-5.3.5.tgz",
+      "integrity": "sha512-VqaplJOVtaGuAxhsw2HM9GG0DLpVi3W9IsV7bKMAC12O7wMIOcZpCYHBw+xkFABzT3xp5MvUqTfbTewCgxgN+A=="
+    },
     "cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "0.2.1",
+  "version": "0.2.6",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -25,6 +25,7 @@
     "cheerio": "^1.0.0-rc.12",
     "concurrently": "^7.6.0",
     "cookies": "^0.8.0",
+    "croner": "^5.3.5",
     "cryptr": "^6.0.3",
     "dayjs": "^1.11.5",
     "dotenv": "^16.0.3",

pages/api/keywords.ts

@@ -153,10 +153,13 @@ const updateKeywords = async (req: NextApiRequest, res: NextApiResponse<Keywords
       }
       if (tags) {
          const tagsKeywordIDs = Object.keys(tags);
+         const multipleKeywords = tagsKeywordIDs.length > 1;
          for (const keywordID of tagsKeywordIDs) {
-            const response = await Keyword.findOne({ where: { ID: keywordID } });
-            if (response) {
-               await response.update({ tags: JSON.stringify(tags[keywordID]) });
+            const selectedKeyword = await Keyword.findOne({ where: { ID: keywordID } });
+            const currentTags = selectedKeyword && selectedKeyword.tags ? JSON.parse(selectedKeyword.tags) : [];
+            const mergedTags = Array.from(new Set([...currentTags, ...tags[keywordID]]));
+            if (selectedKeyword) {
+               await selectedKeyword.update({ tags: JSON.stringify(multipleKeywords ? mergedTags : tags[keywordID]) });
             }
          }
          return res.status(200).json({ keywords });

pages/api/login.ts

@@ -18,10 +18,11 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
    if (!req.body.username || !req.body.password) {
       return res.status(401).json({ error: 'Username Password Missing' });
    }
+   const userName = process.env.USER_NAME ? process.env.USER_NAME : process.env.USER;
 
-   if (req.body.username === process.env.USER
+   if (req.body.username === userName
       && req.body.password === process.env.PASSWORD && process.env.SECRET) {
-      const token = jwt.sign({ user: process.env.USER }, process.env.SECRET);
+      const token = jwt.sign({ user: userName }, process.env.SECRET);
       const cookies = new Cookies(req, res);
       const expireDate = new Date();
       const sessDuration = process.env.SESSION_DURATION;
@@ -30,7 +31,7 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
       return res.status(200).json({ success: true, error: null });
    }
 
-   const error = req.body.username !== process.env.USER ? 'Incorrect Username' : 'Incorrect Password';
+   const error = req.body.username !== userName ? 'Incorrect Username' : 'Incorrect Password';
 
    return res.status(401).json({ success: false, error });
 };

pages/api/settings.ts

@@ -1,5 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import Cryptr from 'cryptr';
+import getConfig from 'next/config';
 import { writeFile, readFile } from 'fs/promises';
 import verifyUser from '../../utils/verifyUser';
 import allScrapers from '../../scrapers/index';
@@ -26,7 +27,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 const getSettings = async (req: NextApiRequest, res: NextApiResponse<SettingsGetResponse>) => {
    const settings = await getAppSettings();
    if (settings) {
-      return res.status(200).json({ settings });
+      const { publicRuntimeConfig } = getConfig();
+      const version = publicRuntimeConfig?.version;
+      return res.status(200).json({ settings: { ...settings, version } });
    }
    return res.status(400).json({ error: 'Error Loading Settings!' });
 };

pages/domains/index.tsx

@@ -11,16 +11,43 @@ import { useFetchDomains } from '../../services/domains';
 import DomainItem from '../../components/domains/DomainItem';
 import Icon from '../../components/common/Icon';
 
+type thumbImages = { [domain:string] : string }
+
 const SingleDomain: NextPage = () => {
    const router = useRouter();
    const [noScrapprtError, setNoScrapprtError] = useState(false);
    const [showSettings, setShowSettings] = useState(false);
    const [showAddDomain, setShowAddDomain] = useState(false);
+   const [domainThumbs, setDomainThumbs] = useState<thumbImages>({});
    const { data: appSettings } = useFetchSettings();
    const { data: domainsData, isLoading } = useFetchDomains(router, true);
 
    useEffect(() => {
-      console.log('Domains Data: ', domainsData);
+      // console.log('Domains Data: ', domainsData);
+      if (domainsData?.domains && domainsData.domains.length > 0) {
+         const domainThumbsRaw = localStorage.getItem('domainThumbs');
+         const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            if (domain.domain) {
+               if (!domThumbs[domain.domain]) {
+                  const domainImageBlob = await fetch(`https://image.thum.io/get/auth/66909-serpbear/maxAge/96/width/200/https://${domain.domain}`).then((res) => res.blob());
+                  if (domainImageBlob) {
+                     const reader = new FileReader();
+                     await new Promise((resolve, reject) => {
+                       reader.onload = resolve;
+                       reader.onerror = reject;
+                       reader.readAsDataURL(domainImageBlob);
+                     });
+                     const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
+                     localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain.domain]: imageBase }));
+                     setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: imageBase }));
+                  }
+               } else {
+                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domThumbs[domain.domain] }));
+               }
+            }
+         });
+      }
    }, [domainsData]);
 
    useEffect(() => {
@@ -31,7 +58,7 @@ const SingleDomain: NextPage = () => {
    }, [appSettings]);
 
    return (
-      <div className="Domain ">
+      <div className="Domain flex flex-col min-h-screen">
          {noScrapprtError && (
                <div className=' p-3 bg-red-600 text-white text-sm text-center'>
                   A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
@@ -62,6 +89,7 @@ const SingleDomain: NextPage = () => {
                            domain={domain}
                            selected={false}
                            isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
+                           thumb={domainThumbs[domain.domain]}
                            // isConsoleIntegrated={false}
                            />;
                })}
@@ -84,6 +112,9 @@ const SingleDomain: NextPage = () => {
          <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
              <Settings closeSettings={() => setShowSettings(false)} />
          </CSSTransition>
+         <footer className='text-center flex flex-1 justify-center pb-5 items-end'>
+            <span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings?.settings?.version || '0.0.0'}</a></span>
+         </footer>
       </div>
    );
 };

scrapers/index.ts

@@ -2,6 +2,7 @@ import scrapingAnt from './services/scrapingant';
 import scrapingRobot from './services/scrapingrobot';
 import serpapi from './services/serpapi';
 import serply from './services/serply';
+import spaceserp from './services/spaceserp';
 import proxy from './services/proxy';
 
 export default [
@@ -9,5 +10,6 @@ export default [
    scrapingAnt,
    serpapi,
    serply,
+   spaceserp,
    proxy,
 ];

scrapers/services/scrapingant.ts

@@ -2,6 +2,11 @@ const scrapingAnt:ScraperSettings = {
    id: 'scrapingant',
    name: 'ScrapingAnt',
    website: 'scrapingant.com',
+   headers: (keyword) => {
+      // eslint-disable-next-line max-len
+      const mobileAgent = 'Mozilla/5.0 (Linux; Android 10; SM-G996U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36';
+      return keyword && keyword.device === 'mobile' ? { 'Ant-User-Agent': mobileAgent } : {};
+   },
    scrapeURL: (keyword, settings, countryData) => {
       const scraperCountries = ['AE', 'BR', 'CN', 'DE', 'ES', 'FR', 'GB', 'HK', 'PL', 'IN', 'IT', 'IL', 'JP', 'NL', 'RU', 'SA', 'US', 'CZ'];
       const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';

scrapers/services/spaceserp.ts

@@ -0,0 +1,34 @@
+interface SpaceSerpResult {
+   title: string,
+   link: string,
+   domain: string,
+   position: number
+}
+
+const spaceSerp:ScraperSettings = {
+   id: 'spaceSerp',
+   name: 'Space Serp',
+   website: 'spaceserp.com',
+   scrapeURL: (keyword, settings, countryData) => {
+      const country = keyword.country || 'US';
+      const lang = countryData[country][2];
+      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${keyword.device === 'mobile' ? '&device=mobile' : ''}&resultBlocks=`;
+   },
+   resultObjectKey: 'organic_results',
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: SpaceSerpResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SpaceSerpResult[];
+      for (const result of results) {
+         if (result.title && result.link) {
+            extractedResult.push({
+               title: result.title,
+               url: result.link,
+               position: result.position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default spaceSerp;

styles/globals.css

@@ -84,8 +84,8 @@ body {
 
 .domKeywords_head--alpha_desc .domKeywords_head_keyword::after,
 .domKeywords_head--pos_desc .domKeywords_head_position::after,
-.domKeywords_head--imp_desc .domKeywords_head_imp::after,
-.domKeywords_head--visits_desc .domKeywords_head_visits::after,
+.domKeywords_head--imp_asc .domKeywords_head_imp::after,
+.domKeywords_head--visits_asc .domKeywords_head_visits::after,
 .domKeywords_head--ctr_desc .domKeywords_head_ctr::after {
    content: "↓";
    display: inline-block;
@@ -98,8 +98,8 @@ body {
 
 .domKeywords_head--alpha_asc .domKeywords_head_keyword::after,
 .domKeywords_head--pos_asc .domKeywords_head_position::after,
-.domKeywords_head--imp_asc .domKeywords_head_imp::after,
-.domKeywords_head--visits_asc .domKeywords_head_visits::after,
+.domKeywords_head--imp_desc .domKeywords_head_imp::after,
+.domKeywords_head--visits_desc .domKeywords_head_visits::after,
 .domKeywords_head--ctr_asc .domKeywords_head_ctr::after {
    content: "↑";
    display: inline-block;

types.d.ts

@@ -78,7 +78,10 @@ type SettingsType = {
    smtp_username?: string,
    smtp_password?: string,
    search_console_integrated?: boolean,
-   available_scapers?: Array
+   available_scapers?: Array,
+   scrape_interval?: string,
+   scrape_delay?: string,
+   version?: string
 }
 
 type KeywordSCDataChild = {

utils/SCsortFilter.ts

@@ -9,16 +9,16 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
    const keywords = theKeywords.map((k) => ({ ...k, position: k.position === 0 ? 111 : k.position }));
    switch (sortBy) {
       case 'imp_asc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.impressions - a.impressions);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.impressions > b.impressions ? 1 : -1));
             break;
       case 'imp_desc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.impressions - b.impressions);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.impressions > a.impressions ? 1 : -1));
             break;
       case 'visits_asc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.clicks - a.clicks);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.clicks > b.clicks ? 1 : -1));
             break;
       case 'visits_desc':
-            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.clicks - b.clicks);
+            sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.clicks > a.clicks ? 1 : -1));
             break;
        case 'ctr_asc':
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.ctr - a.ctr);
@@ -27,17 +27,17 @@ export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCK
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.ctr - b.ctr);
             break;
       case 'pos_asc':
-            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position > a.position ? 1 : -1));
+            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position < a.position ? 1 : -1));
             sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
             break;
       case 'pos_desc':
-            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position > b.position ? 1 : -1));
+            sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position < b.position ? 1 : -1));
             sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position }));
             break;
-      case 'alpha_asc':
+      case 'alpha_desc':
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.keyword > a.keyword ? 1 : -1));
             break;
-      case 'alpha_desc':
+      case 'alpha_asc':
             sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.keyword > b.keyword ? 1 : -1));
          break;
       default:

utils/refresh.ts

@@ -1,4 +1,5 @@
 import { performance } from 'perf_hooks';
+import { setTimeout as sleep } from 'timers/promises';
 import { RefreshResult, scrapeKeywordFromGoogle } from './scraper';
 
 /**
@@ -21,6 +22,9 @@ const refreshKeywords = async (keywords:KeywordType[], settings:SettingsType): P
          console.log('START SCRAPE: ', keyword.keyword);
          const refreshedkeywordData = await scrapeKeywordFromGoogle(keyword, settings);
          refreshedResults.push(refreshedkeywordData);
+         if (keywords.length > 0 && settings.scrape_delay && settings.scrape_delay !== '0') {
+            await sleep(parseInt(settings.scrape_delay, 10));
+         }
       }
    }
 

utils/scraper.ts

@@ -49,7 +49,7 @@ export const getScraperClient = (keyword:KeywordType, settings:SettingsType, scr
       // Set Scraper Header
       const scrapeHeaders = scraper.headers ? scraper.headers(keyword, settings) : null;
       const scraperAPIURL = scraper.scrapeURL ? scraper.scrapeURL(keyword, settings, countries) : null;
-      if (scrapeHeaders) {
+      if (scrapeHeaders && Object.keys(scrapeHeaders).length > 0) {
          Object.keys(scrapeHeaders).forEach((headerItemKey:string) => {
             headers[headerItemKey] = scrapeHeaders[headerItemKey as keyof object];
          });
@@ -114,7 +114,7 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
       const scraperResult = scraperObj?.resultObjectKey && res[scraperObj.resultObjectKey] ? res[scraperObj.resultObjectKey] : '';
       const scrapeResult:string = (res.data || res.html || res.results || scraperResult || '');
       if (res && scrapeResult) {
-         const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult);
+         const extracted = scraperObj?.serpExtractor ? scraperObj.serpExtractor(scrapeResult) : extractScrapedResult(scrapeResult, keyword.device);
          // await writeFile('result.txt', JSON.stringify(scrapeResult), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
          const serp = getSerp(keyword.domain, extracted);
          refreshedResults = { ID: keyword.ID, keyword: keyword.keyword, position: serp.postion, url: serp.url, result: extracted, error: false };
@@ -141,9 +141,10 @@ export const scrapeKeywordFromGoogle = async (keyword:KeywordType, settings:Sett
 /**
  * Extracts the Google Search result as object array from the Google Search's HTML content
  * @param {string} content - scraped google search page html data.
+ * @param {string} device - The device of the keyword.
  * @returns {SearchResult[]}
  */
-export const extractScrapedResult = (content: string): SearchResult[] => {
+export const extractScrapedResult = (content: string, device: string): SearchResult[] => {
    const extractedResult = [];
 
    const $ = cheerio.load(content);
@@ -163,6 +164,24 @@ export const extractScrapedResult = (content: string): SearchResult[] => {
       }
   }
 
+  // Mobile Scraper
+  if (extractedResult.length === 0 && device === 'mobile') {
+      const items = $('body').find('#rso > div');
+      for (let i = 0; i < items.length; i += 1) {
+         const item = $(items[i]);
+         const linkDom = item.find('a[role="presentation"]');
+         if (linkDom) {
+            const url = linkDom.attr('href');
+            const titleDom = linkDom.find('[role="link"]');
+            const title = titleDom ? titleDom.text() : '';
+            if (title && url) {
+               lastPosition += 1;
+               extractedResult.push({ title, url, position: lastPosition });
+            }
+         }
+      }
+   }
+
   return extractedResult;
 };
 
@@ -193,7 +212,7 @@ export const retryScrape = async (keywordID: number) : Promise<void> => {
 
    const filePath = `${process.cwd()}/data/failed_queue.json`;
    const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
-   currentQueue = JSON.parse(currentQueueRaw);
+   currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
 
    if (!currentQueue.includes(keywordID)) {
       currentQueue.push(keywordID);
@@ -213,7 +232,7 @@ export const removeFromRetryQueue = async (keywordID: number) : Promise<void> =>
 
    const filePath = `${process.cwd()}/data/failed_queue.json`;
    const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });
-   currentQueue = JSON.parse(currentQueueRaw);
+   currentQueue = currentQueueRaw ? JSON.parse(currentQueueRaw) : [];
    currentQueue = currentQueue.filter((item) => item !== keywordID);
 
    await writeFile(filePath, JSON.stringify(currentQueue), { encoding: 'utf-8' }).catch((err) => { console.log(err); return '[]'; });

utils/searchConsole.ts

@@ -54,7 +54,7 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
                date: row.keys[0],
                clicks: row.clicks,
                impressions: row.impressions,
-               ctr: row.ctr,
+               ctr: row.ctr * 100,
                position: row.position,
             });
          });
@@ -101,7 +101,7 @@ export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainNam
    const page = SCItem.keys[3] ? SCItem.keys[3].replace('https://', '').replace('http://', '').replace('www', '').replace(domainName, '') : '';
    const uid = `${country.toLowerCase()}:${device}:${keyword.replaceAll(' ', '_')}`;
 
-   return { keyword, uid, device, country, clicks, impressions, ctr, position, page };
+   return { keyword, uid, device, country, clicks, impressions, ctr: ctr * 100, position, page };
 };
 
 export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainDataType) : KeywordType => {

utils/sortFilter.ts

@@ -28,7 +28,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
       case 'alpha_desc':
             sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.keyword > b.keyword ? 1 : -1));
          break;
-      case 'imp_asc':
+      case 'imp_desc':
             if (scDataType) {
                   sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
                   const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
@@ -37,7 +37,7 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
                });
             }
             break;
-      case 'imp_desc':
+      case 'imp_asc':
             if (scDataType) {
                   sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
                   const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0;
@@ -46,21 +46,21 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
                });
             }
          break;
-      case 'visits_asc':
-            if (scDataType) {
-                  sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
-                  const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  return aImpressionData > bImpressionData ? 1 : -1;
-               });
-            }
-            break;
       case 'visits_desc':
             if (scDataType) {
                   sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
-                  const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
-                  return bImpressionData > aImpressionData ? 1 : -1;
+                  const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  return aVisitsData > bVisitsData ? 1 : -1;
+               });
+            }
+            break;
+      case 'visits_asc':
+            if (scDataType) {
+                  sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {
+                  const bVisitsData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  const aVisitsData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0;
+                  return bVisitsData > aVisitsData ? 1 : -1;
                });
             }
             break;