feat: Adds the ability for city level scraping for scapers that allow it.

- Only available for scrapers that allows custom location or city level scraping. - When a city level keyword is added the city name is displayed in the keyword title. closes #139, #151
2025-06-26 18:15:54 +00:00 · 2024-02-06 13:22:32 +06:00
parent 444ba5d461
commit 3719f21d98
11 changed files with 103 additions and 51 deletions
--- a/components/common/Icon.tsx
+++ b/components/common/Icon.tsx
@@ -253,6 +253,17 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
                  <path d="M15.75 9h3v2.25h-3z" fill={color} />
               </svg>
         }
+         {type === 'city'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 48 48">
+                  <g fill="none">
+                     <path stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth={4} d="M4 42h40"></path>
+                     <rect width={8} height={16} x={8} y={26} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path stroke={color} strokeLinecap="square" strokeLinejoin="round" strokeWidth={4} d="M12 34h1"></path>
+                     <rect width={24} height={38} x={16} y={4} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path fill={color} d="M22 10h4v4h-4zm8 0h4v4h-4zm-8 7h4v4h-4zm8 0h4v4h-4zm0 7h4v4h-4zm0 7h4v4h-4z"></path>
+                  </g>
+               </svg>
+         }
       </span>
   );
 };
--- a/components/keywords/AddKeywords.tsx
+++ b/components/keywords/AddKeywords.tsx
@@ -7,6 +7,8 @@ import { useAddKeywords } from '../../services/keywords';

 type AddKeywordsProps = {
   keywords: KeywordType[],
+   scraperName: string,
+   allowsCity: boolean,
   closeModal: Function,
   domain: string
 }
@@ -17,9 +19,10 @@ type KeywordsInput = {
   country: string,
   domain: string,
   tags: string,
+   city?:string,
 }

-const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
+const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCity = false }: AddKeywordsProps) => {
   const [error, setError] = useState<string>('');
   const defCountry = localStorage.getItem('default_country') || 'US';
   const [newKeywordsData, setNewKeywordsData] = useState<KeywordsInput>({ keywords: '', device: 'desktop', country: defCountry, domain, tags: '' });
@@ -29,14 +32,16 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
   const addKeywords = () => {
      if (newKeywordsData.keywords) {
         const keywordsArray = [...new Set(newKeywordsData.keywords.split('\n').map((item) => item.trim()).filter((item) => !!item))];
-         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}`);
-         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(`${k}-${newKeywordsData.device}-${newKeywordsData.country}`));
+         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}${k.city ? `-${k.city}` : ''}`);
+         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(
+            `${k}-${newKeywordsData.device}-${newKeywordsData.country}${newKeywordsData.city ? `-${newKeywordsData.city}` : ''}`,
+         ));
         if (keywordExist.length > 0) {
            setError(`Keywords ${keywordExist.join(',')} already Exist`);
            setTimeout(() => { setError(''); }, 3000);
         } else {
-            const { device, country, domain: kDomain, tags } = newKeywordsData;
-            const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags }));
+            const { device, country, domain: kDomain, tags, city } = newKeywordsData;
+            const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags, city }));
            addMutate(newKeywordsArray);
         }
      } else {
@@ -85,17 +90,28 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
                        ><Icon type='mobile' /> <i className='not-italic hidden lg:inline-block'>Mobile</i></li>
                  </ul>
               </div>
-
               <div className='relative'>
                  {/* TODO:  Insert Existing Tags as Suggestions */}
                  <input
                     className='w-full border rounded border-gray-200 py-2 px-4 pl-8 outline-none focus:border-indigo-300'
-                     placeholder='Insert Tags'
+                     placeholder='Insert Tags (Optional)'
                     value={newKeywordsData.tags}
                     onChange={(e) => setNewKeywordsData({ ...newKeywordsData, tags: e.target.value })}
                  />
                  <span className='absolute text-gray-400 top-2 left-2'><Icon type="tags" size={16} /></span>
               </div>
+               <div className='relative mt-2'>
+                  <input
+                     className={`w-full border rounded border-gray-200 py-2 px-4 pl-8 
+                     outline-none focus:border-indigo-300 ${!allowsCity ? ' cursor-not-allowed' : ''} `}
+                     disabled={!allowsCity}
+                     title={!allowsCity ? `Your scraper ${scraperName} doesn't have city level scraping feature.` : ''}
+                     placeholder={`City (Optional)${!allowsCity ? `. Not avaialable for ${scraperName}.` : ''}`}
+                     value={newKeywordsData.city}
+                     onChange={(e) => setNewKeywordsData({ ...newKeywordsData, city: e.target.value })}
+                  />
+                  <span className='absolute text-gray-400 top-2 left-2'><Icon type="city" size={16} /></span>
+               </div>
            </div>
            {error && <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{error}</div>}
            <div className='mt-6 text-right text-sm font-semibold flex justify-between'>
--- a/components/keywords/Keyword.tsx
+++ b/components/keywords/Keyword.tsx
@@ -40,7 +40,7 @@ const Keyword = (props: KeywordProps) => {
      scDataType = 'threeDays',
   } = props;
   const {
-      keyword, domain, ID, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
+      keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
   } = keywordData;
   const [showOptions, setShowOptions] = useState(false);
   const [showPositionError, setPositionError] = useState(false);
@@ -85,12 +85,12 @@ const Keyword = (props: KeywordProps) => {

   return (
      <div
-      key={keyword}
+      key={keyword + ID}
      style={style}
      className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
      lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>

-         <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold cursor-pointer'>
+         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
            <button
               className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
               ${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
@@ -99,9 +99,10 @@ const Keyword = (props: KeywordProps) => {
                  <Icon type="check" size={10} />
            </button>
            <a
-            className='py-2 hover:text-blue-600'
+            className='py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full'
            onClick={() => showKeywordDetails()}>
-               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />{keyword}
+               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />
+               <span className=' text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-30px)]'>{keyword}{city ? ` (${city})` : ''}</span>
            </a>
            {sticky && <button className='ml-2 relative top-[2px]' title='Favorite'><Icon type="star-filled" size={16} color="#fbd346" /></button>}
            {lastUpdateError && lastUpdateError.date
--- a/components/keywords/KeywordsTable.tsx
+++ b/components/keywords/KeywordsTable.tsx
@@ -1,8 +1,6 @@
 import React, { useState, useMemo } from 'react';
 import { Toaster } from 'react-hot-toast';
-import { CSSTransition } from 'react-transition-group';
 import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
-import AddKeywords from './AddKeywords';
 import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/client/sortFilter';
 import Icon from '../common/Icon';
 import Keyword from './Keyword';
@@ -25,7 +23,7 @@ type KeywordsTableProps = {
 }

 const KeywordsTable = (props: KeywordsTableProps) => {
-   const { domain, keywords = [], isLoading = true, showAddModal = false, setShowAddModal, isConsoleIntegrated = false } = props;
+   const { keywords = [], isLoading = true, isConsoleIntegrated = false } = props;
   const showSCData = isConsoleIntegrated;
   const [device, setDevice] = useState<string>('desktop');
   const [selectedKeywords, setSelectedKeywords] = useState<number[]>([]);
@@ -243,13 +241,6 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                  </div>
            </Modal>
         )}
-         <CSSTransition in={showAddModal} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddKeywords
-               domain={domain?.domain || ''}
-               keywords={keywords}
-               closeModal={() => setShowAddModal(false)}
-               />
-         </CSSTransition>
         {showTagManager && (
            <KeywordTagManager
               allTags={allDomainTags}
--- a/pages/api/keywords.ts
+++ b/pages/api/keywords.ts
@@ -79,13 +79,14 @@ const addKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
      const keywordsToAdd: any = []; // QuickFIX for bug: https://github.com/sequelize/sequelize-typescript/issues/936

      keywords.forEach((kwrd: KeywordAddPayload) => {
-         const { keyword, device, country, domain, tags } = kwrd;
+         const { keyword, device, country, domain, tags, city } = kwrd;
         const tagsArray = tags ? tags.split(',').map((item:string) => item.trim()) : [];
         const newKeyword = {
            keyword,
            device,
            domain,
            country,
+            city,
            position: 0,
            updating: true,
            history: JSON.stringify({}),
--- a/pages/api/settings.ts
+++ b/pages/api/settings.ts
@@ -72,7 +72,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
            scaping_api,
            smtp_password,
            search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL),
-            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
+            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id, allowsCity: !!scraper.allowsCity })),
            failed_queue: failedQueue,
            screenshot_key: screenshotAPIKey,
         };
--- a/pages/domain/[slug]/index.tsx
+++ b/pages/domain/[slug]/index.tsx
@@ -1,9 +1,7 @@
-import React, { useEffect, useMemo, useState } from 'react';
+import React, { useMemo, useState } from 'react';
 import type { NextPage } from 'next';
 import Head from 'next/head';
 import { useRouter } from 'next/router';
-// import { useQuery } from 'react-query';
-// import toast from 'react-hot-toast';
 import { CSSTransition } from 'react-transition-group';
 import Sidebar from '../../../components/common/Sidebar';
 import TopBar from '../../../components/common/TopBar';
@@ -16,17 +14,20 @@ import Settings from '../../../components/settings/Settings';
 import { useFetchDomains } from '../../../services/domains';
 import { useFetchKeywords } from '../../../services/keywords';
 import { useFetchSettings } from '../../../services/settings';
+import AddKeywords from '../../../components/keywords/AddKeywords';

 const SingleDomain: NextPage = () => {
   const router = useRouter();
-   const [noScrapprtError, setNoScrapprtError] = useState(false);
   const [showAddKeywords, setShowAddKeywords] = useState(false);
   const [showAddDomain, setShowAddDomain] = useState(false);
   const [showDomainSettings, setShowDomainSettings] = useState(false);
   const [showSettings, setShowSettings] = useState(false);
   const [keywordSPollInterval, setKeywordSPollInterval] = useState<undefined|number>(undefined);
-   const { data: appSettings } = useFetchSettings();
+   const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings();
   const { data: domainsData } = useFetchDomains(router);
+   const appSettings: SettingsType = appSettingsData?.settings || {};
+   const { scraper_type = '', available_scapers = [] } = appSettings;
+   const activeScraper = useMemo(() => available_scapers.find((scraper) => scraper.value === scraper_type), [scraper_type, available_scapers]);

   const activDomain: DomainType|null = useMemo(() => {
      let active:DomainType|null = null;
@@ -40,18 +41,9 @@ const SingleDomain: NextPage = () => {
   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
   const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;

-   useEffect(() => {
-      // console.log('appSettings.settings: ', appSettings && appSettings.settings);
-      if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) {
-         setNoScrapprtError(true);
-      }
-   }, [appSettings]);
-
-   // console.log('Websites Data:', router, activDomain, theKeywords);
-
   return (
      <div className="Domain ">
-         {noScrapprtError && (
+         {((!scraper_type || (scraper_type === 'none')) && !isAppSettingsLoading) && (
               <div className=' p-3 bg-red-600 text-white text-sm text-center'>
                  A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
               </div>
@@ -80,7 +72,7 @@ const SingleDomain: NextPage = () => {
               keywords={theKeywords}
               showAddModal={showAddKeywords}
               setShowAddModal={setShowAddKeywords}
-               isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
+               isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) }
               />
            </div>
         </div>
@@ -98,6 +90,15 @@ const SingleDomain: NextPage = () => {
         <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
             <Settings closeSettings={() => setShowSettings(false)} />
         </CSSTransition>
+         <CSSTransition in={showAddKeywords} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddKeywords
+               domain={activDomain?.domain || ''}
+               scraperName={activeScraper?.label || ''}
+               keywords={theKeywords}
+               allowsCity={!!activeScraper?.allowsCity}
+               closeModal={() => setShowAddKeywords(false)}
+               />
+         </CSSTransition>
      </div>
   );
 };
--- a/scrapers/services/searchapi.ts
+++ b/scrapers/services/searchapi.ts
@@ -1,7 +1,16 @@
+import countries from '../../utils/countries';
+
+interface SearchApiResult {
+   title: string,
+   link: string,
+   position: number,
+ }
+
 const searchapi:ScraperSettings = {
  id: 'searchapi',
  name: 'SearchApi.io',
  website: 'searchapi.io',
+  allowsCity: true,
  headers: (keyword, settings) => {
     return {
        'Content-Type': 'application/json',
@@ -9,7 +18,10 @@ const searchapi:ScraperSettings = {
     };
  },
  scrapeURL: (keyword) => {
-     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}`;
+   const country = keyword.country || 'US';
+   const countryName = countries[country][0];
+   const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
  },
  resultObjectKey: 'organic_results',
  serpExtractor: (content) => {
@@ -29,10 +41,4 @@ const searchapi:ScraperSettings = {
  },
 };

-interface SearchApiResult {
-  title: string,
-  link: string,
-  position: number,
-}
-
 export default searchapi;
--- a/scrapers/services/serpapi.ts
+++ b/scrapers/services/serpapi.ts
@@ -1,3 +1,5 @@
+import countries from '../../utils/countries';
+
 interface SerpApiResult {
   title: string,
   link: string,
@@ -8,6 +10,7 @@ const serpapi:ScraperSettings = {
   id: 'serpapi',
   name: 'SerpApi.com',
   website: 'serpapi.com',
+   allowsCity: true,
   headers: (keyword, settings) => {
      return {
         'Content-Type': 'application/json',
@@ -15,7 +18,9 @@ const serpapi:ScraperSettings = {
      };
   },
   scrapeURL: (keyword, settings) => {
-      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}&api_key=${settings.scaping_api}`;
+      const countryName = countries[keyword.country || 'US'][0];
+      const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
   },
   resultObjectKey: 'organic_results',
   serpExtractor: (content) => {
--- a/scrapers/services/spaceserp.ts
+++ b/scrapers/services/spaceserp.ts
@@ -1,3 +1,5 @@
+import countries from '../../utils/countries';
+
 interface SpaceSerpResult {
   title: string,
   link: string,
@@ -9,10 +11,14 @@ const spaceSerp:ScraperSettings = {
   id: 'spaceSerp',
   name: 'Space Serp',
   website: 'spaceserp.com',
+   allowsCity: true,
   scrapeURL: (keyword, settings, countryData) => {
      const country = keyword.country || 'US';
+      const countryName = countries[country][0];
+      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const device = keyword.device === 'mobile' ? '&device=mobile' : '';
      const lang = countryData[country][2];
-      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${keyword.device === 'mobile' ? '&device=mobile' : ''}&resultBlocks=`;
+      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
   },
   resultObjectKey: 'organic_results',
   serpExtractor: (content) => {
--- a/types.d.ts
+++ b/types.d.ts
@@ -39,6 +39,7 @@ type KeywordType = {
   lastUpdateError: {date: string, error: string, scraper: string} | false,
   scData?: KeywordSCData,
   uid?: string
+   city?: string
 }

 type KeywordLastResult = {
@@ -78,7 +79,7 @@ type SettingsType = {
   smtp_username?: string,
   smtp_password?: string,
   search_console_integrated?: boolean,
-   available_scapers?: Array,
+   available_scapers?: { label: string, value: string, allowsCity?: boolean }[],
   scrape_interval?: string,
   scrape_delay?: string,
   scrape_retry?: boolean,
@@ -108,7 +109,8 @@ type KeywordAddPayload = {
   device: string,
   country: string,
   domain: string,
-   tags: string,
+   tags?: string,
+   city?:string
 }

 type SearchAnalyticsRawItem = {
@@ -177,11 +179,23 @@ type scraperExtractedItem = {
   position: number,
 }
 interface ScraperSettings {
+   /** A Unique ID for the Scraper. eg: myScraper */
   id:string,
+   /** The Name of the Scraper */
   name:string,
+   /** The Website address of the Scraper */
   website:string,
+   /** The result object's key that contains the results of the scraped data. For example,
+    * if your scraper API the data like this `{scraped:[item1,item2..]}` the resultObjectKey should be "scraped" */
   resultObjectKey: string,
+   /** If the Scraper allows setting a perices location or allows city level scraping set this to true. */
+   allowsCity?: boolean,
+   /** Set your own custom HTTP header properties when making the scraper API request.
+    * The function should return an object that contains all the header properties you want to pass to API request's header.
+    * Example: `{'Cache-Control': 'max-age=0', 'Content-Type': 'application/json'}` */
   headers?(keyword:KeywordType, settings: SettingsType): Object,
+   /** Construct the API URL for scraping the data through your Scraper's API */
   scrapeURL?(keyword:KeywordType, settings:SettingsType, countries:countryData): string,
+   /** Custom function to extract the serp result from the scraped data. The extracted data should be @return {scraperExtractedItem[]} */
   serpExtractor?(content:string): scraperExtractedItem[],
 }