chore(release): 1.0.2

closes #164

.eslintrc.json

@@ -12,6 +12,8 @@
    "no-await-in-loop": "off",
    "arrow-body-style":"off",
    "max-len": ["error", {"code": 150, "ignoreComments": true, "ignoreUrls": true}],
+   "import/no-extraneous-dependencies": "off",
+   "no-unused-vars": "off",
    "import/extensions": [
       "error",
       "ignorePackages",

.sequelizerc

@@ -0,0 +1,8 @@
+const path = require('path');
+
+module.exports = {
+  'config': path.resolve('database', 'config.js'),
+  'models-path': path.resolve('database', 'models'),
+  'seeders-path': path.resolve('database', 'seeders'),
+  'migrations-path': path.resolve('database', 'migrations')
+};

CHANGELOG.md

@@ -2,6 +2,74 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.0.2](https://github.com/towfiqi/serpbear/compare/v1.0.1...v1.0.2) (2024-02-15)
+
+
+### Bug Fixes
+
+* Resolves Incorrect Position issue. ([0e64b95](https://github.com/towfiqi/serpbear/commit/0e64b95cd5303525535ea84a77181281d7f5618e)), closes [#164](https://github.com/towfiqi/serpbear/issues/164)
+
+### [1.0.1](https://github.com/towfiqi/serpbear/compare/v1.0.0...v1.0.1) (2024-02-13)
+
+
+### Bug Fixes
+
+* Resolves the app crash issue when there is no database. ([e5dd411](https://github.com/towfiqi/serpbear/commit/e5dd411aa9aef58ebb226f2b793a2632ab9069a7)), closes [#161](https://github.com/towfiqi/serpbear/issues/161) [#162](https://github.com/towfiqi/serpbear/issues/162)
+
+## [1.0.0](https://github.com/towfiqi/serpbear/compare/v0.3.4...v1.0.0) (2024-02-09)
+
+
+### Features
+
+* Adds Serper.dev integration ([b4ad69b](https://github.com/towfiqi/serpbear/commit/b4ad69baaa0f865938f8b0eace6732a9e6b1b381)), closes [#138](https://github.com/towfiqi/serpbear/issues/138)
+* Adds the ability for city level scraping for scapers that allow it. ([3719f21](https://github.com/towfiqi/serpbear/commit/3719f21d98d173219cef5656579fa0e5340ccdbf)), closes [#139](https://github.com/towfiqi/serpbear/issues/139) [#151](https://github.com/towfiqi/serpbear/issues/151)
+* adds the ability to add url as a domain. ([3c2a1b8](https://github.com/towfiqi/serpbear/commit/3c2a1b8a5b8a2a4a2179a5031582f8202c2e494a)), closes [#53](https://github.com/towfiqi/serpbear/issues/53) [#90](https://github.com/towfiqi/serpbear/issues/90) [#119](https://github.com/towfiqi/serpbear/issues/119)
+* Adds the Ability to set Search Console Property type via Domain Settings. ([b2e97b2](https://github.com/towfiqi/serpbear/commit/b2e97b2ebec380f0edf7ddc0640c2126eff006ac)), closes [#50](https://github.com/towfiqi/serpbear/issues/50)
+* Adds the ability to setup Search Console through the UI. ([f04b10c](https://github.com/towfiqi/serpbear/commit/f04b10cf6b065e3023965112a60e0aa702212a4b)), closes [#59](https://github.com/towfiqi/serpbear/issues/59) [#146](https://github.com/towfiqi/serpbear/issues/146)
+* Adds ValueSerp Integration. ([1041cb3](https://github.com/towfiqi/serpbear/commit/1041cb3c0bb69e9034696624e03433be28e83ac6)), closes [#105](https://github.com/towfiqi/serpbear/issues/105) [#106](https://github.com/towfiqi/serpbear/issues/106)
+
+
+### Bug Fixes
+
+* Resolves Keywords filter crashing issue. ([633ab2c](https://github.com/towfiqi/serpbear/commit/633ab2c467be5b7b86d4547ae0c59034e595a42d))
+* Resolves missing Keyword Loading Spinner issue. ([dbf540c](https://github.com/towfiqi/serpbear/commit/dbf540cfdb16ddb02c9d26618e3680d34799f57f))
+
+### [0.3.4](https://github.com/towfiqi/serpbear/compare/v0.3.3...v0.3.4) (2024-01-15)
+
+
+### Features
+
+* adds ability to add multiple domains at once. ([faa88c9](https://github.com/towfiqi/serpbear/commit/faa88c92542194f19b5cfe2b5cfd07d7d4f7ee46))
+* Adds the ability to show/hide Keys & Passwords in Settings Panel ([c897a52](https://github.com/towfiqi/serpbear/commit/c897a525509baf5b9e8df18d82f5e87aec64f66e))
+
+
+### Bug Fixes
+
+* fixes local SC data not being removed on deleting domain. ([cca9f95](https://github.com/towfiqi/serpbear/commit/cca9f95358b2d3ea06edb33595cdbf616a175469))
+* Resolves incorrect keyword average SC data values in Tracker ([e166b58](https://github.com/towfiqi/serpbear/commit/e166b588aa6c8db55d61b5bc13db66514575c745))
+* resolves newly added Domain's Update time rendering issue ([df3a738](https://github.com/towfiqi/serpbear/commit/df3a738788fa957e7246a0feefe395a9eadd5baf))
+
+### [0.3.3](https://github.com/towfiqi/serpbear/compare/v0.3.2...v0.3.3) (2023-11-12)
+
+
+### Features
+
+* Adds ability to visit pages from Insight tab ([60c68bd](https://github.com/towfiqi/serpbear/commit/60c68bd339db7aeed35aea035dd21691702ffee3))
+* Domains now show their favicon. ([2339e31](https://github.com/towfiqi/serpbear/commit/2339e31af9e90bf918f5bcd4f23114f38cef0313)), closes [#130](https://github.com/towfiqi/serpbear/issues/130)
+* Shows total keywords count in domains page ([fbd23ed](https://github.com/towfiqi/serpbear/commit/fbd23ede256062c72ec2f7e3983a0a02f0240725))
+
+
+### Bug Fixes
+
+* Resolves Website Thumbnail missing issue ([4a60271](https://github.com/towfiqi/serpbear/commit/4a60271cac1209dc02748c4d31943bb21c9ecaf2)), closes [#131](https://github.com/towfiqi/serpbear/issues/131)
+
+### [0.3.2](https://github.com/towfiqi/serpbear/compare/v0.3.1...v0.3.2) (2023-11-09)
+
+
+### Bug Fixes
+
+* Resolves issue with adding long tld emails ([9b9b74a](https://github.com/towfiqi/serpbear/commit/9b9b74af4c249e27458d29ba052e96ab2db8b640)), closes [#127](https://github.com/towfiqi/serpbear/issues/127)
+
 ### [0.3.1](https://github.com/towfiqi/serpbear/compare/v0.3.0...v0.3.1) (2023-11-04)
 
 

Dockerfile

@@ -11,7 +11,8 @@ FROM node:lts-alpine AS builder
 WORKDIR /app
 COPY --from=deps /app ./
 RUN rm -rf /app/data
-RUN rm -rf /app/__test__
+RUN rm -rf /app/__tests__
+RUN rm -rf /app/__mocks__
 RUN npm run build
 
 
@@ -29,13 +30,17 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 # setup the cron
 COPY --from=builder --chown=nextjs:nodejs /app/cron.js ./
 COPY --from=builder --chown=nextjs:nodejs /app/email ./email
+COPY --from=builder --chown=nextjs:nodejs /app/database ./database
+COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc
+COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv croner @googleapis/searchconsole
+RUN npm i cryptr dotenv croner @googleapis/searchconsole sequelize-cli
 RUN npm i -g concurrently
 
 USER nextjs
 
 EXPOSE 3000
 
+ENTRYPOINT ["/app/entrypoint.sh"]
 CMD ["concurrently","node server.js", "node cron.js"]

README.md

@@ -18,7 +18,7 @@ SerpBear is an Open Source Search Engine Position Tracking App. It allows you to
 -   **Zero Cost to RUN:** Run the App on mogenius.com or Fly.io for free.
 
 #### How it Works
-The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SerpApi or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. Also, When you connect your Googel Search Console account, the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages.
+The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. Also, When you connect your Googel Search Console account, the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages.
 
 #### Getting Started
 -   **Step 1:** Deploy & Run the App.
@@ -41,6 +41,8 @@ The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SerpA
 | serply.io | $49/mo | 5000/mo | Yes |
 | serpapi.com | From $50/mo** | From 5,000/mo** | Yes |
 | spaceserp.com | $59/lifetime | 15,000/mo | Yes |
+| SearchApi.io | From $40/mo | From 10,000/mo | Yes |
+| valueserp.com | Pay As You Go | $2.50/1000 req | No |
 
 (*) Free upto a limit. If you are using ScrapingAnt you can lookup 10,000 times per month for free.
 (**) Free up to 100 per month. Paid from 5,000 to 10,000,000+ per month.

__mocks__/data.ts

@@ -2,10 +2,11 @@ export const dummyDomain = {
    ID: 1,
    domain: 'compressimage.io',
    slug: 'compressimage-io',
-   keywordCount: 0,
+   keywordCount: 10,
+   avgPosition: 24,
    lastUpdated: '2022-11-11T10:00:32.243',
    added: '2022-11-11T10:00:32.244',
-   tags: [],
+   tags: '',
    notification: true,
    notification_interval: 'daily',
    notification_emails: '',
@@ -33,7 +34,7 @@ export const dummyKeywords = [
        lastResult: [],
        sticky: false,
        updating: false,
-       lastUpdateError: 'false',
+       lastUpdateError: false as false,
    },
    {
        ID: 2,
@@ -56,6 +57,23 @@ export const dummyKeywords = [
        lastResult: [],
        sticky: false,
        updating: false,
-       lastUpdateError: 'false',
+       lastUpdateError: false as false,
    },
 ];
+
+export const dummySettings = {
+   scaping_api: '',
+   scraper_type: 'none',
+   notification_interval: 'never',
+   notification_email: '',
+   notification_email_from: '',
+   smtp_server: '',
+   smtp_port: '',
+   smtp_username: '',
+   smtp_password: '',
+   scrape_retry: false,
+   search_console_integrated: false,
+   screenshot_key: '',
+   available_scapers: [],
+   failed_queue: [],
+};

__test__/components/Modal.test.tsx

@@ -1,35 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import React from 'react';
-import Modal from '../../components/common/Modal';
-
-// jest.mock('React', () => ({
-//    ...jest.requireActual('React'),
-//    useEffect: jest.fn(),
-// }));
-// const mockedUseEffect = useEffect as jest.Mock<any>;
-
-// jest.mock('../../components/common/Icon', () => () => <div data-testid="icon"/>);
-
-describe('Modal Component', () => {
-   it('Renders without crashing', async () => {
-       render(<Modal closeModal={() => console.log() }><div></div></Modal>);
-      //  mockedUseEffect.mock.calls[0]();
-       expect(document.querySelector('.modal')).toBeInTheDocument();
-   });
-//    it('Sets up the escapae key shortcut', async () => {
-//       render(<Modal closeModal={() => console.log() }><div></div></Modal>);
-//       expect(mockedUseEffect).toBeCalled();
-//   });
-   it('Displays the Given Content', async () => {
-      render(<Modal closeModal={() => console.log() }>
-        <div>
-           <h1>Hello Modal!!</h1>
-        </div>
-      </Modal>);
-      expect(await screen.findByText('Hello Modal!!')).toBeInTheDocument();
-   });
-   it('Renders Modal Title', async () => {
-      render(<Modal closeModal={() => console.log() } title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
-      expect(await screen.findByText('Sample Modal Title')).toBeInTheDocument();
-   });
-});

__test__/components/Sidebar.test.tsx

@@ -1,11 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import Sidebar from '../../components/common/Sidebar';
-
-describe('Sidebar Component', () => {
-   it('renders without crashing', async () => {
-       render(<Sidebar domains={[]} showAddModal={() => console.log() } />);
-       expect(
-           await screen.findByText('SerpBear'),
-       ).toBeInTheDocument();
-   });
-});

__test__/hooks/domains.tests.tsx

@@ -1,21 +0,0 @@
-import { waitFor } from '@testing-library/react';
-// import { useFetchDomains } from '../../services/domains';
-// import { createWrapper } from '../utils';
-
-jest.mock('next/router', () => ({
-   useRouter: () => ({
-     query: { slug: 'compressimage-io' },
-     push: (link:string) => { console.log('Pushed', link); },
-   }),
-}));
-
-describe('DomainHooks', () => {
-   it('useFetchDomains should fetch the Domains', async () => {
-      // const { result } = renderHook(() => useFetchDomains(), { wrapper: createWrapper() });
-      const result = { current: { isSuccess: false, data: '' } };
-      await waitFor(() => {
-         console.log('result.current: ', result.current.data);
-         return expect(result.current.isSuccess).toBe(true);
-      });
-   });
-});

__tests__/components/DomainItem.test.tsx

@@ -0,0 +1,35 @@
+import { fireEvent, render } from '@testing-library/react';
+import DomainItem from '../../components/domains/DomainItem';
+import { dummyDomain } from '../../__mocks__/data';
+
+const updateThumbMock = jest.fn();
+const domainItemProps = {
+   domain: dummyDomain,
+   selected: false,
+   isConsoleIntegrated: false,
+   thumb: '',
+   updateThumb: updateThumbMock,
+};
+
+describe('DomainItem Component', () => {
+   it('renders without crashing', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      expect(container.querySelector('.domItem')).toBeInTheDocument();
+   });
+   it('renders keywords count', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const domStatskeywords = container.querySelector('.dom_stats div:nth-child(1)');
+      expect(domStatskeywords?.textContent).toBe('Keywords10');
+   });
+   it('renders avg position', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const domStatsAvg = container.querySelector('.dom_stats div:nth-child(2)');
+      expect(domStatsAvg?.textContent).toBe('Avg position24');
+   });
+   it('updates domain thumbnail on relevant button click', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const reloadThumbbBtn = container.querySelector('.domain_thumb button');
+      if (reloadThumbbBtn) fireEvent.click(reloadThumbbBtn);
+      expect(updateThumbMock).toHaveBeenCalledWith(dummyDomain.domain);
+   });
+});

__tests__/components/Keyword.test.tsx

@@ -1,8 +1,14 @@
 import { fireEvent, render, screen } from '@testing-library/react';
 import Keyword from '../../components/keywords/Keyword';
-import { dummyKeywords } from '../data';
+import { dummyKeywords } from '../../__mocks__/data';
 
-const keywordFunctions = {
+const keywordProps = {
+   keywordData: dummyKeywords[0],
+   selected: false,
+   index: 0,
+   showSCData: false,
+   scDataType: '',
+   style: {},
    refreshkeyword: jest.fn(),
    favoriteKeyword: jest.fn(),
    removeKeyword: jest.fn(),
@@ -10,35 +16,37 @@ const keywordFunctions = {
    manageTags: jest.fn(),
    showKeywordDetails: jest.fn(),
 };
-
+jest.mock('react-chartjs-2', () => ({
+   Line: () => null,
+ }));
 describe('Keyword Component', () => {
    it('renders without crashing', async () => {
-       render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+       render(<Keyword {...keywordProps} />);
        expect(await screen.findByText('compress image')).toBeInTheDocument();
    });
    it('Should Render Position Correctly', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_position');
       expect(positionElement?.childNodes[0].nodeValue).toBe('19');
    });
    it('Should Display Position Change arrow', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_position i');
       expect(positionElement?.textContent).toBe('▲ 1');
    });
    it('Should Display the SERP Page URL', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_url');
       expect(positionElement?.textContent).toBe('/');
    });
    it('Should Display the Keyword Options on dots Click', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
-      const button = document.querySelector('.keyword .keyword_dots');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      const { container } = render(<Keyword {...keywordProps} />);
+      const button = container.querySelector('.keyword_dots');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.keyword_options')).toBeVisible();
    });
    // it('Should favorite Keywords', async () => {
-   //    render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+   //    render(<Keyword {...keywordProps} />);
    //    const button = document.querySelector('.keyword .keyword_dots');
    //    if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
    //    const option = document.querySelector('.keyword .keyword_options li:nth-child(1) a');

__tests__/components/Modal.test.tsx

@@ -0,0 +1,28 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import Modal from '../../components/common/Modal';
+
+const closeModalMock = jest.fn();
+describe('Modal Component', () => {
+   it('Renders without crashing', async () => {
+       render(<Modal closeModal={closeModalMock }><div></div></Modal>);
+       expect(document.querySelector('.modal')).toBeInTheDocument();
+   });
+   it('Displays the Given Content', async () => {
+      render(<Modal closeModal={closeModalMock}>
+        <div>
+           <h1>Hello Modal!!</h1>
+        </div>
+      </Modal>);
+      expect(await screen.findByText('Hello Modal!!')).toBeInTheDocument();
+   });
+   it('Renders Modal Title', async () => {
+      render(<Modal closeModal={closeModalMock} title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
+      expect(await screen.findByText('Sample Modal Title')).toBeInTheDocument();
+   });
+   it('Closes the modal on close button click', async () => {
+      const { container } = render(<Modal closeModal={closeModalMock} title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
+      const closeBtn = container.querySelector('.modal-close');
+      if (closeBtn) fireEvent.click(closeBtn);
+      expect(closeModalMock).toHaveBeenCalled();
+   });
+});

__tests__/components/Sidebar.test.tsx

@@ -0,0 +1,23 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import Sidebar from '../../components/common/Sidebar';
+import { dummyDomain } from '../../__mocks__/data';
+
+const addDomainMock = jest.fn();
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+
+describe('Sidebar Component', () => {
+   it('renders without crashing', async () => {
+       render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+       expect(screen.getByText('SerpBear')).toBeInTheDocument();
+   });
+   it('renders domain list', async () => {
+      render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+      expect(screen.getByText('compressimage.io')).toBeInTheDocument();
+   });
+   it('calls showAddModal on Add Domain button click', async () => {
+      render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+      const addDomainBtn = screen.getByTestId('add_domain');
+      fireEvent.click(addDomainBtn);
+      expect(addDomainMock).toHaveBeenCalledWith(true);
+   });
+});

__tests__/components/Topbar.test.tsx

@@ -1,9 +1,15 @@
 import { render, screen } from '@testing-library/react';
 import TopBar from '../../components/common/TopBar';
 
+jest.mock('next/router', () => ({
+   useRouter: () => ({
+      pathname: '/',
+   }),
+}));
+
 describe('TopBar Component', () => {
    it('renders without crashing', async () => {
-       render(<TopBar showSettings={() => console.log() } />);
+       render(<TopBar showSettings={jest.fn} showAddModal={jest.fn} />);
        expect(
            await screen.findByText('SerpBear'),
        ).toBeInTheDocument();

__tests__/hooks/domains.test.tsx

@@ -0,0 +1,27 @@
+import { renderHook, waitFor } from '@testing-library/react';
+import mockRouter from 'next-router-mock';
+
+import { useFetchDomains } from '../../services/domains';
+import { createWrapper } from '../../__mocks__/utils';
+import { dummyDomain } from '../../__mocks__/data';
+
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+
+fetchMock.mockIf(`${window.location.origin}/api/domains`, async () => {
+   return new Promise((resolve) => {
+      resolve({
+         body: JSON.stringify({ domains: [dummyDomain] }),
+         status: 200,
+      });
+   });
+});
+
+describe('DomainHooks', () => {
+   it('useFetchDomains should fetch the Domains', async () => {
+      const { result } = renderHook(() => useFetchDomains(mockRouter), { wrapper: createWrapper() });
+      // const result = { current: { isSuccess: false, data: '' } };
+      await waitFor(() => {
+         return expect(result.current.isLoading).toBe(false);
+      });
+   });
+});

__tests__/pages/domain.test.tsx

@@ -1,17 +1,26 @@
 import { fireEvent, render, screen } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from 'react-query';
 import SingleDomain from '../../pages/domain/[slug]';
 import { useAddDomain, useDeleteDomain, useFetchDomains, useUpdateDomain } from '../../services/domains';
-import { useAddKeywords, useDeleteKeywords, useFavKeywords, useFetchKeywords, useRefreshKeywords } from '../../services/keywords';
-import { dummyDomain, dummyKeywords } from '../data';
+import { useAddKeywords, useDeleteKeywords,
+   useFavKeywords, useFetchKeywords, useRefreshKeywords, useFetchSingleKeyword } from '../../services/keywords';
+import { dummyDomain, dummyKeywords, dummySettings } from '../../__mocks__/data';
+import { useFetchSettings } from '../../services/settings';
 
 jest.mock('../../services/domains');
 jest.mock('../../services/keywords');
+jest.mock('../../services/settings');
+
 jest.mock('next/router', () => ({
    useRouter: () => ({
      query: { slug: dummyDomain.slug },
    }),
 }));
 
+jest.mock('react-chartjs-2', () => ({
+   Line: () => null,
+}));
+
 const useFetchDomainsFunc = useFetchDomains as jest.Mock<any>;
 const useFetchKeywordsFunc = useFetchKeywords as jest.Mock<any>;
 const useDeleteKeywordsFunc = useDeleteKeywords as jest.Mock<any>;
@@ -21,11 +30,17 @@ const useAddDomainFunc = useAddDomain as jest.Mock<any>;
 const useAddKeywordsFunc = useAddKeywords as jest.Mock<any>;
 const useUpdateDomainFunc = useUpdateDomain as jest.Mock<any>;
 const useDeleteDomainFunc = useDeleteDomain as jest.Mock<any>;
+const useFetchSettingsFunc = useFetchSettings as jest.Mock<any>;
+const useFetchSingleKeywordFunc = useFetchSingleKeyword as jest.Mock<any>;
 
 describe('SingleDomain Page', () => {
+   const queryClient = new QueryClient();
    beforeEach(() => {
+      useFetchSettingsFunc.mockImplementation(() => ({ data: { settings: dummySettings }, isLoading: false }));
       useFetchDomainsFunc.mockImplementation(() => ({ data: { domains: [dummyDomain] }, isLoading: false }));
       useFetchKeywordsFunc.mockImplementation(() => ({ keywordsData: { keywords: dummyKeywords }, keywordsLoading: false }));
+      const fetchPayload = { history: dummyKeywords[0].history || [], searchResult: dummyKeywords[0].lastResult || [] };
+      useFetchSingleKeywordFunc.mockImplementation(() => ({ data: fetchPayload, isLoading: false }));
       useDeleteKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
       useFavKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
       useRefreshKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
@@ -38,158 +53,154 @@ describe('SingleDomain Page', () => {
       jest.clearAllMocks();
    });
    it('Render without crashing.', async () => {
-      const { getByTestId } = render(<SingleDomain />);
-      // screen.debug(undefined, Infinity);
-      expect(getByTestId('domain-header')).toBeInTheDocument();
-      // expect(await result.findByText(/compressimage/i)).toBeInTheDocument();
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      expect(screen.getByTestId('domain-header')).toBeInTheDocument();
    });
    it('Should Call the useFetchDomains hook on render.', async () => {
-      render(<SingleDomain />);
-      // screen.debug(undefined, Infinity);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       expect(useFetchDomains).toHaveBeenCalled();
-      // expect(await result.findByText(/compressimage/i)).toBeInTheDocument();
    });
    it('Should Render the Keywords', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(2);
    });
    it('Should Display the Keywords Details Sidebar on Keyword Click.', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const keywords = document.querySelectorAll('.keyword');
       const firstKeyword = keywords && keywords[0].querySelector('a');
-      if (firstKeyword) fireEvent(firstKeyword, new MouseEvent('click', { bubbles: true }));
+      if (firstKeyword) fireEvent.click(firstKeyword);
+      expect(useFetchSingleKeyword).toHaveBeenCalled();
       expect(screen.getByTestId('keywordDetails')).toBeVisible();
    });
    it('Should Display the AddDomain Modal on Add Domain Button Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=add_domain]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('add_domain');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('adddomain_modal')).toBeVisible();
    });
    it('Should Display the AddKeywords Modal on Add Keyword Button Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=add_keyword]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('add_keyword');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('addkeywords_modal')).toBeVisible();
    });
 
    it('Should display the Domain Settings on Settings Button click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=show_domain_settings]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('show_domain_settings');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('domain_settings')).toBeVisible();
    });
 
    it('Device Tab change should be functioning.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=mobile_tab]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('mobile_tab');
+      if (button) fireEvent.click(button);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(0);
    });
 
    it('Search Filter should function properly', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const inputNode = screen.getByTestId('filter_input');
-      fireEvent.change(inputNode, { target: { value: 'compressor' } }); // triggers onChange event
+      if (inputNode) fireEvent.change(inputNode, { target: { value: 'compressor' } }); // triggers onChange event
       expect(inputNode.getAttribute('value')).toBe('compressor');
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(1);
    });
 
    it('Country Filter should function properly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=filter_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('filter_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.country_filter')).toBeVisible();
 
       const countrySelect = document.querySelector('.country_filter .selected');
-      if (countrySelect) fireEvent(countrySelect, new MouseEvent('click', { bubbles: true }));
+      if (countrySelect) fireEvent.click(countrySelect);
       expect(document.querySelector('.country_filter .select_list')).toBeVisible();
       const firstCountry = document.querySelector('.country_filter .select_list ul li:nth-child(1)');
-      if (firstCountry) fireEvent(firstCountry, new MouseEvent('click', { bubbles: true }));
+      if (firstCountry) fireEvent.click(firstCountry);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(0);
    });
 
    // Tags Filter should function properly
    it('Tags Filter should Render & Function properly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=filter_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('filter_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.tags_filter')).toBeVisible();
 
       const countrySelect = document.querySelector('.tags_filter .selected');
-      if (countrySelect) fireEvent(countrySelect, new MouseEvent('click', { bubbles: true }));
+      if (countrySelect) fireEvent.click(countrySelect);
       expect(document.querySelector('.tags_filter .select_list')).toBeVisible();
       expect(document.querySelectorAll('.tags_filter .select_list ul li').length).toBe(1);
 
       const firstTag = document.querySelector('.tags_filter .select_list ul li:nth-child(1)');
-      if (firstTag) fireEvent(firstTag, new MouseEvent('click', { bubbles: true }));
+      if (firstTag) fireEvent.click(firstTag);
       expect(document.querySelectorAll('.keyword').length).toBe(1);
    });
 
    it('Sort Options Should be visible Sort Button on Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.sort_options')).toBeVisible();
    });
 
    it('Sort: Position should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
-
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(1)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(2)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });
 
    it('Sort: Date Added should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
 
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(3)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(4)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });
 
    it('Sort: Alphabetical should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
 
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(5)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(6)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });

__tests__/pages/domains.test.tsx

@@ -0,0 +1,49 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from 'react-query';
+import * as ReactQuery from 'react-query';
+import { dummyDomain } from '../../__mocks__/data';
+import Domains from '../../pages/domains';
+
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+jest.spyOn(ReactQuery, 'useQuery').mockImplementation(jest.fn().mockReturnValue(
+   { data: { domains: [dummyDomain] }, isLoading: false, isSuccess: true },
+));
+
+fetchMock.mockIf(`${window.location.origin}/api/domains`, async () => {
+   return new Promise((resolve) => {
+      resolve({
+         body: JSON.stringify({ domains: [dummyDomain] }),
+         status: 200,
+      });
+   });
+});
+
+describe('Domains Page', () => {
+   const queryClient = new QueryClient();
+   it('Renders without crashing', async () => {
+      render(
+          <QueryClientProvider client={queryClient}>
+              <Domains />
+          </QueryClientProvider>,
+      );
+      expect(screen.getByTestId('domains')).toBeInTheDocument();
+   });
+   it('Renders the Domain Component', async () => {
+      const { container } = render(
+          <QueryClientProvider client={queryClient}>
+              <Domains />
+          </QueryClientProvider>,
+      );
+      expect(container.querySelector('.domItem')).toBeInTheDocument();
+   });
+   it('Should Display Add Domain Modal on relveant Button Click.', async () => {
+      render(<QueryClientProvider client={queryClient}><Domains /></QueryClientProvider>);
+      const button = screen.getByTestId('addDomainButton');
+      if (button) fireEvent.click(button);
+      expect(screen.getByTestId('adddomain_modal')).toBeVisible();
+   });
+   it('Should Display the version number in Footer.', async () => {
+      render(<QueryClientProvider client={queryClient}><Domains /></QueryClientProvider>);
+      expect(screen.getByText('SerpBear v0.0.0')).toBeVisible();
+   });
+});

__tests__/pages/index.test.tsx

@@ -2,21 +2,16 @@ import { render, screen } from '@testing-library/react';
 import { QueryClient, QueryClientProvider } from 'react-query';
 import Home from '../../pages/index';
 
+const routerPush = jest.fn();
+jest.mock('next/router', () => ({
+   useRouter: () => ({
+      push: routerPush,
+   }),
+}));
+
 describe('Home Page', () => {
    const queryClient = new QueryClient();
    it('Renders without crashing', async () => {
-      // const dummyDomain = {
-      //    ID: 1,
-      //    domain: 'compressimage.io',
-      //    slug: 'compressimage-io',
-      //    keywordCount: 0,
-      //    lastUpdated: '2022-11-11T10:00:32.243',
-      //    added: '2022-11-11T10:00:32.244',
-      //    tags: [],
-      //    notification: true,
-      //    notification_interval: 'daily',
-      //    notification_emails: '',
-      // };
       render(
           <QueryClientProvider client={queryClient}>
               <Home />
@@ -26,12 +21,12 @@ describe('Home Page', () => {
       expect(await screen.findByRole('main')).toBeInTheDocument();
       expect(screen.queryByText('Add Domain')).not.toBeInTheDocument();
    });
-   it('Should Display the Add Domain Modal when there are no Domains.', async () => {
+   it('Should redirect to /domains route.', async () => {
        render(
            <QueryClientProvider client={queryClient}>
                <Home />
            </QueryClientProvider>,
        );
-       expect(await screen.findByText('Add Domain')).toBeInTheDocument();
+       expect(routerPush).toHaveBeenCalledWith('/domains');
    });
 });

components/common/Icon.tsx

@@ -221,6 +221,14 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
                   </g>
                </svg>
          }
+         {type === 'eye-closed'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <g fill="none" stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth="2">
+                     <path d="M6.873 17.129c-1.845-1.31-3.305-3.014-4.13-4.09a1.693 1.693 0 0 1 0-2.077C4.236 9.013 7.818 5 12 5c1.876 0 3.63.807 5.13 1.874"/>
+                     <path d="M14.13 9.887a3 3 0 1 0-4.243 4.242M4 20L20 4M10 18.704A7.124 7.124 0 0 0 12 19c4.182 0 7.764-4.013 9.257-5.962a1.694 1.694 0 0 0-.001-2.078A22.939 22.939 0 0 0 19.57 9"/>
+                  </g>
+               </svg>
+         }
          {type === 'target'
             && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
                   <path d="M19.938 13A8.004 8.004 0 0 1 13 19.938V22h-2v-2.062A8.004 8.004 0 0 1 4.062 13H2v-2h2.062A8.004 8.004 0 0 1 11 4.062V2h2v2.062A8.004 8.004 0 0 1 19.938 11H22v2h-2.062zM12 18a6 6 0 1 0 0-12a6 6 0 0 0 0 12zm0-3a3 3 0 1 0 0-6a3 3 0 0 0 0 6z" fill={color} fillRule="nonzero"/>
@@ -245,6 +253,27 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
                   <path d="M15.75 9h3v2.25h-3z" fill={color} />
                </svg>
          }
+         {type === 'email'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill={color} d="M22 6c0-1.1-.9-2-2-2H4c-1.1 0-2 .9-2 2v12c0 1.1.9 2 2 2h16c1.1 0 2-.9 2-2zm-2 0l-8 5l-8-5zm0 12H4V8l8 5l8-5z" />
+               </svg>
+         }
+         {type === 'scraper'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 16 16">
+               <path fill={color} d="M1 3.5A2.5 2.5 0 0 1 3.5 1h7A2.5 2.5 0 0 1 13 3.5v1.53a4.538 4.538 0 0 0-1-.004V5H2v5.5A1.5 1.5 0 0 0 3.5 12h2.954l-.72.72a2.52 2.52 0 0 0-.242.28H3.5A2.5 2.5 0 0 1 1 10.5zm7.931 3.224l-.577-.578a.5.5 0 1 0-.708.708l.745.744c.144-.306.324-.6.54-.874M2 4h10v-.5A1.5 1.5 0 0 0 10.5 2h-7A1.5 1.5 0 0 0 2 3.5zm4.354 2.854a.5.5 0 1 0-.708-.708l-2 2a.5.5 0 0 0 0 .708l2 2a.5.5 0 0 0 .708-.708L4.707 8.5zm6.538-.83c.366.042.471.48.21.742l-.975.975a1.507 1.507 0 1 0 2.132 2.132l.975-.975c.261-.261.7-.156.742.21a3.518 3.518 0 0 1-4.676 3.723l-2.726 2.727a1.507 1.507 0 1 1-2.132-2.132L9.168 10.7a3.518 3.518 0 0 1 3.724-4.676" />
+            </svg>
+         }
+         {type === 'city'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 48 48">
+                  <g fill="none">
+                     <path stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth={4} d="M4 42h40"></path>
+                     <rect width={8} height={16} x={8} y={26} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path stroke={color} strokeLinecap="square" strokeLinejoin="round" strokeWidth={4} d="M12 34h1"></path>
+                     <rect width={24} height={38} x={16} y={4} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path fill={color} d="M22 10h4v4h-4zm8 0h4v4h-4zm-8 7h4v4h-4zm8 0h4v4h-4zm0 7h4v4h-4zm0 7h4v4h-4z"></path>
+                  </g>
+               </svg>
+         }
        </span>
    );
  };

components/common/InputField.tsx

@@ -0,0 +1,28 @@
+type InputFieldProps = {
+   label: string;
+   value: string;
+   onChange: Function;
+   placeholder?: string;
+   classNames?: string;
+   hasError?: boolean;
+}
+
+const InputField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: InputFieldProps) => {
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+   return (
+      <div className="field--input w-full relative flex justify-between items-center">
+         <label className={labelStyle}>{label}</label>
+         <input
+            className={`p-2 border border-gray-200 rounded focus:outline-none w-[210px]
+             focus:border-blue-200 ${hasError ? ' border-red-400 focus:border-red-400' : ''} `}
+            type={'text'}
+            value={value}
+            onChange={(event) => onChange(event.target.value)}
+            autoComplete="off"
+            placeholder={placeholder}
+         />
+      </div>
+   );
+};
+
+export default InputField;

components/common/Modal.tsx

@@ -1,25 +1,17 @@
-import React, { useEffect } from 'react';
+import React from 'react';
 import Icon from './Icon';
+import useOnKey from '../../hooks/useOnKey';
 
 type ModalProps = {
    children: React.ReactNode,
    width?: string,
    title?: string,
+   verticalCenter?: boolean,
    closeModal: Function,
 }
 
-const Modal = ({ children, width = '1/2', closeModal, title }:ModalProps) => {
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            closeModal();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeModal]);
+const Modal = ({ children, width = '1/2', closeModal, title, verticalCenter = false }:ModalProps) => {
+   useOnKey('Escape', closeModal);
 
    const closeOnBGClick = (e:React.SyntheticEvent) => {
       e.stopPropagation();
@@ -30,8 +22,9 @@ const Modal = ({ children, width = '1/2', closeModal, title }:ModalProps) => {
    return (
       <div className='modal fixed top-0 left-0 bg-white/[.7] w-full h-screen z-50' onClick={closeOnBGClick}>
          <div
-         className={`modal__content max-w-[340px] absolute top-1/4 left-0 right-0 ml-auto mr-auto w-${width} 
-         lg:max-w-md bg-white shadow-md rounded-md p-5 border-t-[1px] border-gray-100 text-base`}>
+         className={`modal__content max-w-[340px] absolute left-0 right-0 ml-auto mr-auto w-${width} 
+         lg:max-w-md bg-white shadow-md rounded-md p-5 border-t-[1px] border-gray-100 text-base 
+         ${verticalCenter ? ' top-1/2 translate-y-[-50%]' : 'top-1/4'}`}>
             {title && <h3 className=' font-semibold mb-3'>{title}</h3>}
             <button
             className='modal-close absolute right-2 top-2 p-2 cursor-pointer text-gray-400 transition-all

components/common/SecretField.tsx

@@ -0,0 +1,37 @@
+import { useState } from 'react';
+import Icon from './Icon';
+
+type SecretFieldProps = {
+   label: string;
+   value: string;
+   onChange: Function;
+   placeholder?: string;
+   classNames?: string;
+   hasError?: boolean;
+}
+
+const SecretField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: SecretFieldProps) => {
+   const [showValue, setShowValue] = useState(false);
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+   return (
+      <div className="settings__section__secret mb-5 relative flex justify-between items-center">
+         <label className={labelStyle}>{label}</label>
+         <span
+         className="absolute top-1 right-0 px-2 py-1 cursor-pointer text-gray-400 select-none"
+         onClick={() => setShowValue(!showValue)}>
+            <Icon type={showValue ? 'eye-closed' : 'eye'} size={18} />
+         </span>
+         <input
+            className={`w-[210px] p-2 border border-gray-200 rounded focus:outline-none 
+             focus:border-blue-200 ${hasError ? ' border-red-400 focus:border-red-400' : ''} `}
+            type={showValue ? 'text' : 'password'}
+            value={value}
+            onChange={(event) => onChange(event.target.value)}
+            autoComplete="off"
+            placeholder={placeholder}
+         />
+      </div>
+   );
+};
+
+export default SecretField;

components/common/SelectField.tsx

@@ -9,6 +9,7 @@ type SelectFieldProps = {
    defaultLabel: string,
    options: SelectionOption[],
    selected: string[],
+   label?: string,
    multiple?: boolean,
    updateField: Function,
    minWidth?: number,
@@ -28,6 +29,7 @@ const SelectField = (props: SelectFieldProps) => {
       maxHeight = 96,
       rounded = 'rounded-3xl',
       flags = false,
+      label = '',
       emptyMsg = '' } = props;
 
    const [showOptions, setShowOptions] = useState<boolean>(false);
@@ -66,12 +68,13 @@ const SelectField = (props: SelectFieldProps) => {
    };
 
    return (
-       <div className="select font-semibold text-gray-500">
+       <div className="select font-semibold text-gray-500 relative flex justify-between items-center">
+         {label && <label className='mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'>{label}</label>}
          <div
-         className={`selected flex border ${rounded} p-1.5 px-4 cursor-pointer select-none w-[180px] min-w-[${minWidth}px] 
+         className={`selected flex border ${rounded} p-1.5 px-4 cursor-pointer select-none w-[210px] min-w-[${minWidth}px] 
          ${showOptions ? 'border-indigo-200' : ''}`}
          onClick={() => setShowOptions(!showOptions)}>
-            <span className={`w-[${minWidth - 30}px] inline-block truncate mr-2 capitalize`}>
+            <span className={'w-full inline-block truncate mr-2 capitalize'}>
                {selected.length > 0 ? (selectedLabels.slice(0, 2).join(', ')) : defaultLabel}
             </span>
             {multiple && selected.length > 2
@@ -80,7 +83,7 @@ const SelectField = (props: SelectFieldProps) => {
          </div>
          {showOptions && (
             <div
-            className={`select_list mt-1 border absolute min-w-[${minWidth}px] 
+            className={`select_list mt-1 border absolute min-w-[${minWidth}px] top-[30px] right-0 w-[210px]
             ${rounded === 'rounded-3xl' ? 'rounded-lg' : rounded} max-h-${maxHeight} bg-white z-50 overflow-y-auto styled-scrollbar`}>
                {options.length > 20 && (
                   <div className=''>

components/common/Sidebar.tsx

@@ -1,3 +1,4 @@
+/* eslint-disable @next/next/no-img-element */
 import React from 'react';
 import Link from 'next/link';
 import { useRouter } from 'next/router';
@@ -26,11 +27,11 @@ const Sidebar = ({ domains, showAddModal } : SidebarProps) => {
                                         rounded-r-none ${((`/domain/${d.slug}` === router.asPath || `/domain/console/${d.slug}` === router.asPath
                                         || `/domain/insight/${d.slug}` === router.asPath)
                                         ? 'bg-white text-zinc-800 border border-r-0' : 'text-zinc-500')}`}>
-                                          <i className={'text-center leading-4 mr-2 inline-block rounded-full w-5 h-5 bg-orange-200 not-italic'}>
-                                             {d.domain.charAt(0)}
-                                          </i>
+                                          <img
+                                          className={' inline-block mr-1'}
+                                          src={`https://www.google.com/s2/favicons?domain=${d.domain}&sz=16`} alt={d.domain}
+                                          />
                                           {d.domain}
-                                          {/* <span>0</span> */}
                                        </a>
                                     </Link>
                                  </li>)

components/common/ToggleField.tsx

@@ -0,0 +1,32 @@
+type ToggleFieldProps = {
+   label: string;
+   value: string;
+   onChange: (bool:boolean) => void ;
+   classNames?: string;
+}
+
+const ToggleField = ({ label = '', value = '', onChange, classNames = '' }: ToggleFieldProps) => {
+   return (
+      <div className={`field--toggle w-full relative ${classNames}`}>
+         <label className="relative inline-flex items-center cursor-pointer w-full justify-between">
+            <span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-56">{label}</span>
+            <input
+            type="checkbox"
+            value={value}
+            checked={!!value}
+            className="sr-only peer"
+            onChange={() => onChange(!value)}
+            />
+            <div className="relative rounded-3xl w-9 h-5 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
+            peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
+            peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
+            after:absolute after:top-[2px] after:left-[2px] after:bg-white  after:border-gray-300
+            after:border after:rounded-full after:h-4 after:w-4
+            after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
+
+         </label>
+      </div>
+   );
+};
+
+export default ToggleField;

components/domains/AddDomain.tsx

@@ -1,54 +1,66 @@
 import React, { useState } from 'react';
 import Modal from '../common/Modal';
 import { useAddDomain } from '../../services/domains';
-import { isValidDomain } from '../../utils/validators';
+import { isValidUrl } from '../../utils/client/validators';
 
 type AddDomainProps = {
+   domains: DomainType[],
    closeModal: Function
 }
 
-const AddDomain = ({ closeModal }: AddDomainProps) => {
+const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
    const [newDomain, setNewDomain] = useState<string>('');
-   const [newDomainError, setNewDomainError] = useState<boolean>(false);
+   const [newDomainError, setNewDomainError] = useState('');
    const { mutate: addMutate, isLoading: isAdding } = useAddDomain(() => closeModal());
 
    const addDomain = () => {
-      // console.log('ADD NEW DOMAIN', newDomain);
-      if (isValidDomain(newDomain.trim())) {
-         setNewDomainError(false);
-         // TODO: Domain Action
-         addMutate(newDomain.trim());
-      } else {
-         setNewDomainError(true);
+      setNewDomainError('');
+      const existingDomains = domains.map((d) => d.domain);
+      const insertedURLs = newDomain.split('\n');
+      const domainsTobeAdded:string[] = [];
+      const invalidDomains:string[] = [];
+      insertedURLs.forEach((url) => {
+        const theURL = url.trim();
+        if (isValidUrl(theURL)) {
+         const domURL = new URL(theURL);
+         const isDomain = domURL.pathname === '/';
+         if (isDomain && !existingDomains.includes(domURL.host)) {
+            domainsTobeAdded.push(domURL.host);
+         }
+         if (!isDomain && !existingDomains.includes(domURL.href)) {
+            const cleanedURL = domURL.href.replace('https://', '').replace('http://', '').replace(/^\/+|\/+$/g, '');
+            domainsTobeAdded.push(cleanedURL);
+         }
+        } else {
+         invalidDomains.push(theURL);
+        }
+      });
+      if (invalidDomains.length > 0) {
+         setNewDomainError(`Please Insert Valid Domain URL. Invalid URLs: ${invalidDomains.join(', ')}`);
+      } else if (domainsTobeAdded.length > 0) {
+            console.log('domainsTobeAdded :', domainsTobeAdded);
+         addMutate(domainsTobeAdded);
       }
    };
 
-   const handleDomainInput = (e:React.FormEvent<HTMLInputElement>) => {
-      if (e.currentTarget.value === '' && newDomainError) { setNewDomainError(false); }
+   const handleDomainInput = (e:React.ChangeEvent<HTMLTextAreaElement>) => {
+      if (e.currentTarget.value === '' && newDomainError) { setNewDomainError(''); }
       setNewDomain(e.currentTarget.value);
    };
 
    return (
       <Modal closeModal={() => { closeModal(false); }} title={'Add New Domain'}>
          <div data-testid="adddomain_modal">
-            <h4 className='text-sm mt-4'>
-               Domain Name {newDomainError && <span className=' ml-2 block float-right text-red-500 text-xs font-semibold'>Not a Valid Domain</span>}
-            </h4>
-            <input
-               className={`w-full p-2 border border-gray-200 rounded mt-2 mb-3 focus:outline-none  focus:border-blue-200 
-               ${newDomainError ? ' border-red-400 focus:border-red-400' : ''} `}
-               type="text"
+            <h4 className='text-sm mt-4'>Domain URL</h4>
+            <textarea
+               className={`w-full h-40 border rounded border-gray-200 p-4 outline-none
+                focus:border-indigo-300 ${newDomainError ? ' border-red-400 focus:border-red-400' : ''}`}
+               placeholder="Type or Paste URLs here. Insert Each URL in a New line."
                value={newDomain}
-               placeholder={'example.com'}
-               onChange={handleDomainInput}
                autoFocus={true}
-               onKeyDown={(e) => {
-                  if (e.code === 'Enter') {
-                    e.preventDefault();
-                    addDomain();
-                  }
-               }}
-            />
+               onChange={handleDomainInput}>
+            </textarea>
+            {newDomainError && <div><span className=' ml-2 block float-right text-red-500 text-xs font-semibold'>{newDomainError}</span></div>}
             <div className='mt-6 text-right text-sm font-semibold'>
                <button className='py-2 px-5 rounded cursor-pointer bg-indigo-50 text-slate-500 mr-3' onClick={() => closeModal(false)}>Cancel</button>
                <button className='py-2 px-5 rounded cursor-pointer bg-blue-700 text-white' onClick={() => !isAdding && addDomain() }>

components/domains/DomainItem.tsx

@@ -11,9 +11,10 @@ type DomainItemProps = {
    selected: boolean,
    isConsoleIntegrated: boolean,
    thumb: string,
+   updateThumb: Function,
 }
 
-const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: DomainItemProps) => {
+const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb, updateThumb }: DomainItemProps) => {
    const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain;
    // const router = useRouter();
    return (
@@ -21,11 +22,23 @@ const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: Do
          <Link href={`/domain/${slug}`} passHref={true}>
          <a className='flex flex-col lg:flex-row'>
             <div className={`flex-1 p-6 flex ${!isConsoleIntegrated ? 'basis-1/3' : ''}`}>
-               <div className="domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded border border-gray-200 overflow-hidden">
-                  {thumb && <img src={thumb} alt={domain.domain} />}
+               <div className="group domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded
+                  border border-gray-200 overflow-hidden flex justify-center relative">
+                  <button
+                     className=' absolute right-1 top-0 text-gray-400 p-1 transition-all
+                     invisible opacity-0 group-hover:visible group-hover:opacity-100 hover:text-gray-600 z-10'
+                     title='Reload Website Screenshot'
+                     onClick={(e) => { e.preventDefault(); e.stopPropagation(); updateThumb(domain.domain); }}
+                  >
+                     <Icon type="reload" size={12} />
+                  </button>
+                  <img
+                  className={`self-center ${!thumb ? 'max-w-[50px]' : ''}`}
+                  src={thumb || `https://www.google.com/s2/favicons?domain=${domain.domain}&sz=128`} alt={domain.domain}
+                  />
                </div>
                <div className="domain_details flex-1">
-                  <h3 className='font-semibold text-base mb-2'>{domain.domain}</h3>
+                  <h3 className='font-semibold text-base mb-2 max-w-[200px] text-ellipsis overflow-hidden' title={domain.domain}>{domain.domain}</h3>
                  {keywordsUpdated && (
                   <span className=' text-gray-600 text-xs'>
                      Updated <TimeAgo title={dayjs(keywordsUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={keywordsUpdated} />

components/domains/DomainSettings.tsx

@@ -1,8 +1,10 @@
 import { useRouter } from 'next/router';
-import { useEffect, useState } from 'react';
+import { useState } from 'react';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
-import { useDeleteDomain, useUpdateDomain } from '../../services/domains';
+import { useDeleteDomain, useFetchDomain, useUpdateDomain } from '../../services/domains';
+import InputField from '../common/InputField';
+import SelectField from '../common/SelectField';
 
 type DomainSettingsProps = {
    domain:DomainType|false,
@@ -16,32 +18,31 @@ type DomainSettingsError = {
 
 const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
    const router = useRouter();
+   const [currentTab, setCurrentTab] = useState<'notification'|'searchconsole'>('notification');
    const [showRemoveDomain, setShowRemoveDomain] = useState<boolean>(false);
    const [settingsError, setSettingsError] = useState<DomainSettingsError>({ type: '', msg: '' });
-   const [domainSettings, setDomainSettings] = useState<DomainSettings>({ notification_interval: 'never', notification_emails: '' });
+   const [domainSettings, setDomainSettings] = useState<DomainSettings>(() => ({
+      notification_interval: domain && domain.notification_interval ? domain.notification_interval : 'never',
+      notification_emails: domain && domain.notification_emails ? domain.notification_emails : '',
+      search_console: domain && domain.search_console ? JSON.parse(domain.search_console) : {
+         property_type: 'domain', url: '', client_email: '', private_key: '',
+      },
+   }));
 
-   const { mutate: updateMutate } = useUpdateDomain(() => closeModal(false));
-   const { mutate: deleteMutate } = useDeleteDomain(() => {
-      closeModal(false);
-      router.push('/domains');
+   const { mutate: updateMutate, error: domainUpdateError, isLoading: isUpdating } = useUpdateDomain(() => closeModal(false));
+   const { mutate: deleteMutate } = useDeleteDomain(() => { closeModal(false); router.push('/domains'); });
+
+   // Get the Full Domain Data along with the Search Console API Data.
+   useFetchDomain(router, domain && domain.domain ? domain.domain : '', (domainObj:DomainType) => {
+      const currentSearchConsoleSettings = domainObj.search_console && JSON.parse(domainObj.search_console);
+      setDomainSettings({ ...domainSettings, search_console: currentSearchConsoleSettings || domainSettings.search_console });
    });
 
-   useEffect(() => {
-      if (domain) {
-         setDomainSettings({ notification_interval: domain.notification_interval, notification_emails: domain.notification_emails });
-      }
-   }, [domain]);
-
-   const updateNotiEmails = (event:React.FormEvent<HTMLInputElement>) => {
-      setDomainSettings({ ...domainSettings, notification_emails: event.currentTarget.value });
-   };
-
    const updateDomain = () => {
-      console.log('Domain: ');
       let error: DomainSettingsError | null = null;
       if (domainSettings.notification_emails) {
          const notification_emails = domainSettings.notification_emails.split(',');
-         const invalidEmails = notification_emails.find((x) => /^\w+([.-]?\w+)*@\w+([.-]?\w+)*(\.\w{2,3})+$/.test(x) === false);
+         const invalidEmails = notification_emails.find((x) => /^\w+([.-]?\w+)*@\w+([.-]?\w+)*(\.\w{2,15})+$/.test(x) === false);
          console.log('invalidEmails: ', invalidEmails);
          if (invalidEmails) {
             error = { type: 'email', msg: 'Invalid Email' };
@@ -58,24 +59,103 @@ const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
          }
    };
 
+   const tabStyle = `inline-block px-4 py-2 rounded-md mr-3 cursor-pointer text-sm select-none z-10
+                     text-gray-600 border border-b-0 relative top-[1px] rounded-b-none`;
    return (
       <div>
-         <Modal closeModal={() => closeModal(false)} title={'Domain Settings'} width="[500px]">
+         <Modal closeModal={() => closeModal(false)} title={'Domain Settings'} width="[500px]" verticalCenter={currentTab === 'searchconsole'} >
             <div data-testid="domain_settings" className=" text-sm">
-               <div className="mb-6 flex justify-between items-center">
-                  <h4>Notification Emails
-                     {settingsError.type === 'email' && <span className="text-red-500 font-semibold ml-2">{settingsError.msg}</span>}
-                  </h4>
-                  <input
-                     className={`border w-46 text-sm transition-all rounded p-1.5 px-4 outline-none ring-0 
-                     ${settingsError.type === 'email' ? ' border-red-300' : ''}`}
-                     type="text"
-                     placeholder='Your Emails'
-                     onChange={updateNotiEmails}
-                     value={domainSettings.notification_emails || ''}
-                  />
+               <div className=' mt-3 mb-5 border  border-slate-200 px-2 py-4 pb-0
+               relative left-[-20px] w-[calc(100%+40px)] border-l-0 border-r-0 bg-[#f8f9ff]'>
+                  <ul>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'notification' ? ' bg-white text-blue-600 border-slate-200' : 'border-transparent'} `}
+                     onClick={() => setCurrentTab('notification')}>
+                       <Icon type='email' /> Notification
+                     </li>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'searchconsole' ? ' bg-white text-blue-600 border-slate-200' : 'border-transparent'}`}
+                     onClick={() => setCurrentTab('searchconsole')}>
+                        <Icon type='google' /> Search Console
+                     </li>
+                  </ul>
                </div>
+
+               <div>
+                  {currentTab === 'notification' && (
+                     <div className="mb-4 flex justify-between items-center w-full">
+                        <InputField
+                        label='Notification Emails'
+                        onChange={(emails:string) => setDomainSettings({ ...domainSettings, notification_emails: emails })}
+                        value={domainSettings.notification_emails || ''}
+                        placeholder='Your Emails'
+                        />
+                     </div>
+                  )}
+                  {currentTab === 'searchconsole' && (
+                     <>
+                        <div className="mb-4 flex justify-between items-center w-full">
+                           <label className='mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'>Property Type</label>
+                           <SelectField
+                           options={[{ label: 'Domain', value: 'domain' }, { label: 'URL', value: 'url' }]}
+                           selected={[domainSettings.search_console?.property_type || 'domain']}
+                           defaultLabel="Select Search Console Property Type"
+                           updateField={(updated:['domain'|'url']) => setDomainSettings({
+                              ...domainSettings,
+                              search_console: { ...(domainSettings.search_console as DomainSearchConsole), property_type: updated[0] || 'domain' },
+                           })}
+                           multiple={false}
+                           rounded={'rounded'}
+                           />
+                        </div>
+                        {domainSettings?.search_console?.property_type === 'url' && (
+                           <div className="mb-4 flex justify-between items-center w-full">
+                              <InputField
+                              label='Property URL (Required)'
+                              onChange={(url:string) => setDomainSettings({
+                                 ...domainSettings,
+                                 search_console: { ...(domainSettings.search_console as DomainSearchConsole), url },
+                              })}
+                              value={domainSettings?.search_console?.url || ''}
+                              placeholder='Search Console Property URL. eg: https://mywebsite.com/'
+                              />
+                           </div>
+                        )}
+                        <div className="mb-4 flex justify-between items-center w-full">
+                           <InputField
+                           label='Search Console Client Email'
+                           onChange={(client_email:string) => setDomainSettings({
+                              ...domainSettings,
+                              search_console: { ...(domainSettings.search_console as DomainSearchConsole), client_email },
+                           })}
+                           value={domainSettings?.search_console?.client_email || ''}
+                           placeholder='myapp@appspot.gserviceaccount.com'
+                           />
+                        </div>
+                        <div className="mb-4 flex flex-col justify-between items-center w-full">
+                           <label className='mb-2 font-semibold block text-sm text-gray-700 capitalize w-full'>Search Console Private Key</label>
+                           <textarea
+                              className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
+                              focus:outline-none h-[100px] focus:border-blue-200`}
+                              value={domainSettings?.search_console?.private_key || ''}
+                              placeholder={'-----BEGIN PRIVATE KEY-----/ssssaswdkihad....'}
+                              onChange={(event) => setDomainSettings({
+                                 ...domainSettings,
+                                 search_console: { ...(domainSettings.search_console as DomainSearchConsole), private_key: event.target.value },
+                              })}
+                           />
+                        </div>
+                     </>
+                  )}
+               </div>
+               {!isUpdating && (domainUpdateError as Error)?.message && (
+                  <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{(domainUpdateError as Error).message}</div>
+               )}
+               {!isUpdating && settingsError?.msg && (
+                  <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{settingsError.msg}</div>
+               )}
             </div>
+
             <div className="flex justify-between border-t-[1px] border-gray-100 mt-8 pt-4 pb-0">
                <button
                className="text-sm font-semibold text-red-500"
@@ -83,9 +163,9 @@ const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
                   <Icon type="trash" /> Remove Domain
                </button>
                <button
-               className='text-sm font-semibold py-2 px-5 rounded cursor-pointer bg-blue-700 text-white'
-               onClick={() => updateDomain()}>
-                  Update Settings
+               className={`text-sm font-semibold py-2 px-5 rounded cursor-pointer bg-blue-700 text-white ${isUpdating ? 'cursor-not-allowed' : ''}`}
+               onClick={() => !isUpdating && updateDomain()}>
+                  {isUpdating && <Icon type='loading' />} Update Settings
                </button>
             </div>
          </Modal>

components/insight/Insight.tsx

@@ -12,7 +12,7 @@ type SCInsightProps = {
    isConsoleIntegrated: boolean,
 }
 
-const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SCInsightProps) => {
+const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true, domain }: SCInsightProps) => {
    const [activeTab, setActiveTab] = useState<string>('stats');
 
    const insightItems = insight[activeTab as keyof InsightDataType];
@@ -108,7 +108,7 @@ const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SC
                               (item:SCInsightItem, index: number) => {
                               const insightItemCount = insight ? insightItems : [];
                               const lastItem = !!(insightItemCount && (index === insightItemCount.length));
-                              return <InsightItem key={index} item={item} type={activeTab} lastItem={lastItem} />;
+                              return <InsightItem key={index} item={item} type={activeTab} lastItem={lastItem} domain={domain?.domain || ''} />;
                            },
                         )
                      }
@@ -117,7 +117,7 @@ const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SC
                      )}
                      {!isConsoleIntegrated && (
                         <p className=' p-9 pt-[10%] text-center text-gray-500'>
-                        Google Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        Google Search Console has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
                         </p>
                      )}
                   </div>

components/insight/InsightItem.tsx

@@ -1,21 +1,22 @@
 import React from 'react';
 import countries from '../../utils/countries';
 import Icon from '../common/Icon';
+import { formattedNum } from '../../utils/client/helpers';
 
 type InsightItemProps = {
    item: SCInsightItem,
    lastItem: boolean,
-   type: string
+   type: string,
+   domain: string
 }
 
-const InsightItem = ({ item, lastItem, type }:InsightItemProps) => {
+const InsightItem = ({ item, lastItem, type, domain }:InsightItemProps) => {
    const { clicks, impressions, ctr, position, country = 'zzz', keyword, page, keywords = 0, countries: cntrs = 0, date } = item;
    let firstItem = keyword;
    if (type === 'pages') { firstItem = page; } if (type === 'stats') {
       firstItem = date && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(date));
    }
    if (type === 'countries') { firstItem = countries[country] && countries[country][0]; }
-   const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num);
 
    return (
       <div
@@ -24,7 +25,7 @@ const InsightItem = ({ item, lastItem, type }:InsightItemProps) => {
 
          <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold'>
             {type === 'countries' && <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} />}
-            {firstItem}
+            {type === 'pages' && domain ? <a href={`https://${domain}${page}`} target='_blank' rel="noreferrer">{firstItem}</a> : firstItem}
          </div>
 
          <div className='keyword_pos text-center inline-block mr-3 lg:mr-0 lg:flex-1'>
@@ -34,7 +35,6 @@ const InsightItem = ({ item, lastItem, type }:InsightItemProps) => {
             {Math.round(position)}
          </div>
 
-         {/* <div className='keyword_imp text-center inline-block lg:flex-1'>{formattedNum(clicks)}</div> */}
          <div className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-14 p-2 text-base mt-[-55px] rounded right-5 lg:relative
           lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-40 lg:grow-0 lg:right-0 text-center font-semibold`}>
             {formattedNum(clicks)}

components/insight/InsightStats.tsx

@@ -1,6 +1,7 @@
-import React, { useMemo, useState, useEffect } from 'react';
+import React, { useMemo } from 'react';
 import { Chart as ChartJS, CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend } from 'chart.js';
 import { Line } from 'react-chartjs-2';
+import { formattedNum } from '../../utils/client/helpers';
 
 ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend);
 
@@ -12,21 +13,15 @@ type InsightStatsProps = {
 }
 
 const InsightStats = ({ stats = [], totalKeywords = 0, totalPages = 0 }:InsightStatsProps) => {
-   const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num);
-   const [totalStat, setTotalStat] = useState({ impressions: 0, clicks: 0, ctr: 0, position: 0 });
-
-   useEffect(() => {
-      if (stats.length > 0) {
-         const totalStats = stats.reduce((acc, item) => {
-            return {
-               impressions: item.impressions + acc.impressions,
-               clicks: item.clicks + acc.clicks,
-               ctr: item.ctr + acc.ctr,
-               position: item.position + acc.position,
-            };
-         }, { impressions: 0, clicks: 0, ctr: 0, position: 0 });
-         setTotalStat(totalStats);
-      }
+   const totalStat = useMemo(() => {
+      return stats.reduce((acc, item) => {
+         return {
+            impressions: item.impressions + acc.impressions,
+            clicks: item.clicks + acc.clicks,
+            ctr: item.ctr + acc.ctr,
+            position: item.position + acc.position,
+         };
+      }, { impressions: 0, clicks: 0, ctr: 0, position: 0 });
    }, [stats]);
 
    const chartData = useMemo(() => {

components/keywords/AddKeywords.tsx

@@ -7,6 +7,8 @@ import { useAddKeywords } from '../../services/keywords';
 
 type AddKeywordsProps = {
    keywords: KeywordType[],
+   scraperName: string,
+   allowsCity: boolean,
    closeModal: Function,
    domain: string
 }
@@ -17,9 +19,10 @@ type KeywordsInput = {
    country: string,
    domain: string,
    tags: string,
+   city?:string,
 }
 
-const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
+const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCity = false }: AddKeywordsProps) => {
    const [error, setError] = useState<string>('');
    const defCountry = localStorage.getItem('default_country') || 'US';
    const [newKeywordsData, setNewKeywordsData] = useState<KeywordsInput>({ keywords: '', device: 'desktop', country: defCountry, domain, tags: '' });
@@ -29,14 +32,16 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
    const addKeywords = () => {
       if (newKeywordsData.keywords) {
          const keywordsArray = [...new Set(newKeywordsData.keywords.split('\n').map((item) => item.trim()).filter((item) => !!item))];
-         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}`);
-         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(`${k}-${newKeywordsData.device}-${newKeywordsData.country}`));
+         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}${k.city ? `-${k.city}` : ''}`);
+         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(
+            `${k}-${newKeywordsData.device}-${newKeywordsData.country}${newKeywordsData.city ? `-${newKeywordsData.city}` : ''}`,
+         ));
          if (keywordExist.length > 0) {
             setError(`Keywords ${keywordExist.join(',')} already Exist`);
             setTimeout(() => { setError(''); }, 3000);
          } else {
-            const { device, country, domain: kDomain, tags } = newKeywordsData;
-            const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags }));
+            const { device, country, domain: kDomain, tags, city } = newKeywordsData;
+            const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags, city }));
             addMutate(newKeywordsArray);
          }
       } else {
@@ -85,17 +90,28 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
                         ><Icon type='mobile' /> <i className='not-italic hidden lg:inline-block'>Mobile</i></li>
                   </ul>
                </div>
-
                <div className='relative'>
                   {/* TODO:  Insert Existing Tags as Suggestions */}
                   <input
                      className='w-full border rounded border-gray-200 py-2 px-4 pl-8 outline-none focus:border-indigo-300'
-                     placeholder='Insert Tags'
+                     placeholder='Insert Tags (Optional)'
                      value={newKeywordsData.tags}
                      onChange={(e) => setNewKeywordsData({ ...newKeywordsData, tags: e.target.value })}
                   />
                   <span className='absolute text-gray-400 top-2 left-2'><Icon type="tags" size={16} /></span>
                </div>
+               <div className='relative mt-2'>
+                  <input
+                     className={`w-full border rounded border-gray-200 py-2 px-4 pl-8 
+                     outline-none focus:border-indigo-300 ${!allowsCity ? ' cursor-not-allowed' : ''} `}
+                     disabled={!allowsCity}
+                     title={!allowsCity ? `Your scraper ${scraperName} doesn't have city level scraping feature.` : ''}
+                     placeholder={`City (Optional)${!allowsCity ? `. Not avaialable for ${scraperName}.` : ''}`}
+                     value={newKeywordsData.city}
+                     onChange={(e) => setNewKeywordsData({ ...newKeywordsData, city: e.target.value })}
+                  />
+                  <span className='absolute text-gray-400 top-2 left-2'><Icon type="city" size={16} /></span>
+               </div>
             </div>
             {error && <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{error}</div>}
             <div className='mt-6 text-right text-sm font-semibold flex justify-between'>

components/keywords/Keyword.tsx

@@ -4,7 +4,8 @@ import dayjs from 'dayjs';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
 import ChartSlim from '../common/ChartSlim';
-import { generateTheChartData } from '../common/generateChartData';
+import KeywordPosition from './KeywordPosition';
+import { generateTheChartData } from '../../utils/client/generateChartData';
 
 type KeywordProps = {
    keywordData: KeywordType,
@@ -39,7 +40,7 @@ const Keyword = (props: KeywordProps) => {
       scDataType = 'threeDays',
    } = props;
    const {
-      keyword, domain, ID, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
+      keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
    } = keywordData;
    const [showOptions, setShowOptions] = useState(false);
    const [showPositionError, setPositionError] = useState(false);
@@ -82,24 +83,14 @@ const Keyword = (props: KeywordProps) => {
 
    const optionsButtonStyle = 'block px-2 py-2 cursor-pointer hover:bg-indigo-50 hover:text-blue-700';
 
-   const renderPosition = (pos:number, type?:string) => {
-      if (!updating && pos === 0) {
-         return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
-      }
-      if (updating && type !== 'sc') {
-         return <span title='Updating Keyword Position'><Icon type="loading" /></span>;
-      }
-      return pos;
-   };
-
    return (
       <div
-      key={keyword}
+      key={keyword + ID}
       style={style}
       className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
       lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
 
-         <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold cursor-pointer'>
+         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
             <button
                className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
                ${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
@@ -108,9 +99,10 @@ const Keyword = (props: KeywordProps) => {
                   <Icon type="check" size={10} />
             </button>
             <a
-            className='py-2 hover:text-blue-600'
+            className='py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full'
             onClick={() => showKeywordDetails()}>
-               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />{keyword}
+               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />
+               <span className=' text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-30px)]'>{keyword}{city ? ` (${city})` : ''}</span>
             </a>
             {sticky && <button className='ml-2 relative top-[2px]' title='Favorite'><Icon type="star-filled" size={16} color="#fbd346" /></button>}
             {lastUpdateError && lastUpdateError.date
@@ -123,7 +115,7 @@ const Keyword = (props: KeywordProps) => {
          <div
          className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative
           lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center font-semibold`}>
-            {renderPosition(position)}
+            <KeywordPosition position={position} updating={updating} />
             {!updating && positionChange > 0 && <i className=' not-italic ml-1 text-xs text-[#5ed7c3]'>▲ {positionChange}</i>}
             {!updating && positionChange < 0 && <i className=' not-italic ml-1 text-xs text-red-300'>▼ {positionChange}</i>}
          </div>
@@ -164,7 +156,10 @@ const Keyword = (props: KeywordProps) => {
             relative flex justify-between text-center lg:flex-1 lg:text-sm lg:m-0 lg:mt-0 lg:border-t-0 lg:pt-0 lg:top-0'>
                <span className='min-w-[40px]'>
                   <span className='lg:hidden'>SC Position: </span>
-                  {renderPosition(keywordData?.scData?.position[scDataType as keyof KeywordSCDataChild] || 0, 'sc')}
+                  <KeywordPosition
+                  position={keywordData?.scData?.position[scDataType as keyof KeywordSCDataChild] || 0}
+                  type='sc'
+                  />
                </span>
                <span className='min-w-[40px]'>
                   <span className='lg:hidden'>Impressions: </span>{keywordData?.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0}

components/keywords/KeywordDetails.tsx

@@ -1,10 +1,12 @@
-import React, { useEffect, useMemo, useRef, useState } from 'react';
+import React, { useLayoutEffect, useMemo, useRef, useState } from 'react';
 import dayjs from 'dayjs';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
 import Chart from '../common/Chart';
 import SelectField from '../common/SelectField';
-import { generateTheChartData } from '../common/generateChartData';
+import { useFetchSingleKeyword } from '../../services/keywords';
+import useOnKey from '../../hooks/useOnKey';
+import { generateTheChartData } from '../../utils/client/generateChartData';
 
 type KeywordDetailsProps = {
    keyword: KeywordType,
@@ -13,11 +15,12 @@ type KeywordDetailsProps = {
 
 const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
    const updatedDate = new Date(keyword.lastUpdated);
-   const [keywordHistory, setKeywordHistory] = useState<KeywordHistory>(keyword.history);
-   const [keywordSearchResult, setKeywordSearchResult] = useState<KeywordLastResult[]>([]);
    const [chartTime, setChartTime] = useState<string>('30');
    const searchResultContainer = useRef<HTMLDivElement>(null);
    const searchResultFound = useRef<HTMLDivElement>(null);
+   const { data: keywordData } = useFetchSingleKeyword(keyword.ID);
+   const keywordHistory: KeywordHistory = keywordData?.history || keyword.history;
+   const keywordSearchResult: KeywordLastResult = keywordData?.searchResult || keyword.history;
    const dateOptions = [
       { label: 'Last 7 Days', value: '7' },
       { label: 'Last 30 Days', value: '30' },
@@ -26,39 +29,9 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
       { label: 'All Time', value: 'all' },
    ];
 
-   useEffect(() => {
-      const fetchFullKeyword = async () => {
-         try {
-            const fetchURL = `${window.location.origin}/api/keyword?id=${keyword.ID}`;
-            const res = await fetch(fetchURL, { method: 'GET' }).then((result) => result.json());
-            if (res.keyword) {
-               console.log(res.keyword, new Date().getTime());
-               setKeywordHistory(res.keyword.history || []);
-               setKeywordSearchResult(res.keyword.lastResult || []);
-            }
-         } catch (error) {
-            console.log(error);
-         }
-      };
-      if (keyword.lastResult.length === 0) {
-         fetchFullKeyword();
-      }
-   }, [keyword]);
+   useOnKey('Escape', closeDetails);
 
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            console.log(event.key);
-            closeDetails();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeDetails]);
-
-   useEffect(() => {
+   useLayoutEffect(() => {
       if (keyword.position < 100 && keyword.position > 0 && searchResultFound?.current) {
          searchResultFound.current.scrollIntoView({
             behavior: 'smooth',

components/keywords/KeywordFilter.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useEffect, useMemo } from 'react';
+import React, { useState, useMemo } from 'react';
 import Icon from '../common/Icon';
 import SelectField, { SelectionOption } from '../common/SelectField';
 import countries from '../../utils/countries';
@@ -17,18 +17,13 @@ type KeywordFilterProps = {
    SCcountries?: string[];
 }
 
-type KeywordCountState = {
-   desktop: number,
-   mobile: number
-}
-
 const KeywordFilters = (props: KeywordFilterProps) => {
    const {
       device,
       setDevice,
       filterKeywords,
       allTags = [],
-      keywords,
+      keywords = [],
       updateSort,
       sortBy,
       filterParams,
@@ -36,20 +31,21 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       integratedConsole = false,
       SCcountries = [],
     } = props;
-   const [keywordCounts, setKeywordCounts] = useState<KeywordCountState>({ desktop: 0, mobile: 0 });
    const [sortOptions, showSortOptions] = useState(false);
    const [filterOptions, showFilterOptions] = useState(false);
 
-   useEffect(() => {
-      const keyWordCount = { desktop: 0, mobile: 0 };
-      keywords.forEach((k) => {
-         if (k.device === 'desktop') {
-            keyWordCount.desktop += 1;
-         } else {
-            keyWordCount.mobile += 1;
-         }
-      });
-      setKeywordCounts(keyWordCount);
+   const keywordCounts = useMemo(() => {
+      const counts = { desktop: 0, mobile: 0 };
+      if (keywords && keywords.length > 0) {
+         keywords.forEach((k) => {
+            if (k.device === 'desktop') {
+               counts.desktop += 1;
+            } else {
+               counts.mobile += 1;
+            }
+         });
+      }
+      return counts;
    }, [keywords]);
 
    const filterCountry = (cntrs:string[]) => filterKeywords({ ...filterParams, countries: cntrs });

components/keywords/KeywordPosition.tsx

@@ -0,0 +1,19 @@
+import Icon from '../common/Icon';
+
+type KeywordPositionProps = {
+   position: number,
+   updating?: boolean,
+   type?: string,
+}
+
+const KeywordPosition = ({ position = 0, type = '', updating = false }:KeywordPositionProps) => {
+   if (!updating && position === 0) {
+      return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
+   }
+   if (updating && type !== 'sc') {
+      return <span title='Updating Keyword Position'><Icon type="loading" /></span>;
+   }
+   return <>{Math.round(position)}</>;
+};
+
+export default KeywordPosition;

components/keywords/KeywordsTable.tsx

@@ -1,9 +1,7 @@
-import React, { useState, useMemo, useEffect } from 'react';
+import React, { useState, useMemo } from 'react';
 import { Toaster } from 'react-hot-toast';
-import { CSSTransition } from 'react-transition-group';
 import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
-import AddKeywords from './AddKeywords';
-import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/sortFilter';
+import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/client/sortFilter';
 import Icon from '../common/Icon';
 import Keyword from './Keyword';
 import KeywordDetails from './KeywordDetails';
@@ -12,6 +10,8 @@ import Modal from '../common/Modal';
 import { useDeleteKeywords, useFavKeywords, useRefreshKeywords } from '../../services/keywords';
 import KeywordTagManager from './KeywordTagManager';
 import AddTags from './AddTags';
+import useWindowResize from '../../hooks/useWindowResize';
+import useIsMobile from '../../hooks/useIsMobile';
 
 type KeywordsTableProps = {
    domain: DomainType | null,
@@ -23,7 +23,7 @@ type KeywordsTableProps = {
 }
 
 const KeywordsTable = (props: KeywordsTableProps) => {
-   const { domain, keywords = [], isLoading = true, showAddModal = false, setShowAddModal, isConsoleIntegrated = false } = props;
+   const { keywords = [], isLoading = true, isConsoleIntegrated = false } = props;
    const showSCData = isConsoleIntegrated;
    const [device, setDevice] = useState<string>('desktop');
    const [selectedKeywords, setSelectedKeywords] = useState<number[]>([]);
@@ -31,7 +31,6 @@ const KeywordsTable = (props: KeywordsTableProps) => {
    const [showRemoveModal, setShowRemoveModal] = useState<boolean>(false);
    const [showTagManager, setShowTagManager] = useState<null|number>(null);
    const [showAddTags, setShowAddTags] = useState<boolean>(false);
-   const [isMobile, setIsMobile] = useState<boolean>(false);
    const [SCListHeight, setSCListHeight] = useState(500);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
    const [sortBy, setSortBy] = useState<string>('date_asc');
@@ -40,26 +39,18 @@ const KeywordsTable = (props: KeywordsTableProps) => {
    const { mutate: deleteMutate } = useDeleteKeywords(() => {});
    const { mutate: favoriteMutate } = useFavKeywords(() => {});
    const { mutate: refreshMutate } = useRefreshKeywords(() => {});
+   const [isMobile] = useIsMobile();
+   useWindowResize(() => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400)));
 
    const scDataObject:{ [k:string] : string} = {
       threeDays: 'Last Three Days',
       sevenDays: 'Last Seven Days',
       thirtyDays: 'Last Thirty Days',
-      avgSevenDays: 'Last Three Days Avg',
-      avgThreeDays: 'Last Seven Days Avg',
+      avgThreeDays: 'Last Three Days Avg',
+      avgSevenDays: 'Last Seven Days Avg',
       avgThirtyDays: 'Last Thirty Days Avg',
    };
 
-   useEffect(() => {
-      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
-      const resizeList = () => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400));
-      resizeList();
-      window.addEventListener('resize', resizeList);
-      return () => {
-          window.removeEventListener('resize', resizeList);
-      };
-   }, [isMobile]);
-
    const processedKeywords: {[key:string] : KeywordType[]} = useMemo(() => {
       const procKeywords = keywords.filter((x) => x.device === device);
       const filteredKeywords = filterKeywords(procKeywords, filterParams);
@@ -250,13 +241,6 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                   </div>
             </Modal>
          )}
-         <CSSTransition in={showAddModal} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddKeywords
-               domain={domain?.domain || ''}
-               keywords={keywords}
-               closeModal={() => setShowAddModal(false)}
-               />
-         </CSSTransition>
          {showTagManager && (
             <KeywordTagManager
                allTags={allDomainTags}

components/keywords/SCKeyword.tsx

@@ -1,6 +1,8 @@
 import React from 'react';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
+import KeywordPosition from './KeywordPosition';
+import { formattedNum } from '../../utils/client/helpers';
 
 type SCKeywordProps = {
    keywordData: SearchAnalyticsItem,
@@ -15,13 +17,6 @@ const SCKeyword = (props: SCKeywordProps) => {
    const { keywordData, selected, lastItem, selectKeyword, style, isTracked = false } = props;
    const { keyword, uid, position, country, impressions, ctr, clicks } = keywordData;
 
-   const renderPosition = () => {
-      if (position === 0) {
-         return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
-      }
-      return Math.round(position);
-   };
-
    return (
       <div
       key={keyword}
@@ -45,7 +40,7 @@ const SCKeyword = (props: SCKeywordProps) => {
 
          <div className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-15 p-2 text-base mt-[-20px] rounded right-5 lg:relative
           lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-40 lg:grow-0 lg:right-0 text-center font-semibold`}>
-            {renderPosition()}
+            <KeywordPosition position={position} />
             <span className='block text-xs text-gray-500 lg:hidden'>Position</span>
          </div>
 
@@ -53,14 +48,14 @@ const SCKeyword = (props: SCKeywordProps) => {
             <span className='mr-3 lg:hidden'>
                <Icon type="eye" size={14} color="#999" />
             </span>
-            {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(impressions)}
+            {formattedNum(impressions)}
          </div>
 
          <div className={'keyword_visits text-center inline-block mt-4 mr-5 ml-5 lg:flex-1 lg:m-0 max-w-[70px] lg:max-w-none lg:pr-5'}>
             <span className='mr-3 lg:hidden'>
                <Icon type="cursor" size={14} color="#999" />
             </span>
-            {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(clicks)}
+            {formattedNum(clicks)}
          </div>
 
          <div className='keyword_ctr text-center inline-block mt-4 relative lg:flex-1 lg:m-0 '>

components/keywords/SCKeywordsTable.tsx

@@ -1,12 +1,15 @@
 import { useRouter } from 'next/router';
-import React, { useState, useMemo, useEffect } from 'react';
+import React, { useState, useMemo } from 'react';
 import { Toaster } from 'react-hot-toast';
 import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
 import { useAddKeywords, useFetchKeywords } from '../../services/keywords';
-import { SCfilterKeywords, SCkeywordsByDevice, SCsortKeywords } from '../../utils/SCsortFilter';
+import { SCfilterKeywords, SCkeywordsByDevice, SCsortKeywords } from '../../utils/client/SCsortFilter';
 import Icon from '../common/Icon';
 import KeywordFilters from './KeywordFilter';
 import SCKeyword from './SCKeyword';
+import useWindowResize from '../../hooks/useWindowResize';
+import useIsMobile from '../../hooks/useIsMobile';
+import { formattedNum } from '../../utils/client/helpers';
 
 type SCKeywordsTableProps = {
    domain: DomainType | null,
@@ -27,11 +30,13 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
    const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
    const [sortBy, setSortBy] = useState<string>('imp_desc');
-   const [isMobile, setIsMobile] = useState<boolean>(false);
    const [SCListHeight, setSCListHeight] = useState(500);
-   const { keywordsData } = useFetchKeywords(router);
+   const { keywordsData } = useFetchKeywords(router, domain?.domain || '');
    const addedkeywords: string[] = keywordsData?.keywords?.map((key: KeywordType) => `${key.keyword}:${key.country}:${key.device}`) || [];
    const { mutate: addKeywords } = useAddKeywords(() => { if (domain && domain.slug) router.push(`/domain/${domain.slug}`); });
+   const [isMobile] = useIsMobile();
+   useWindowResize(() => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400)));
+
    const finalKeywords: {[key:string] : SCKeywordType[] } = useMemo(() => {
       const procKeywords = keywords.filter((x) => x.device === device);
       const filteredKeywords = SCfilterKeywords(procKeywords, filterParams);
@@ -71,16 +76,6 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
          };
    }, [finalKeywords, device]);
 
-   useEffect(() => {
-      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
-      const resizeList = () => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400));
-      resizeList();
-      window.addEventListener('resize', resizeList);
-      return () => {
-          window.removeEventListener('resize', resizeList);
-      };
-   }, [isMobile]);
-
    const selectKeyword = (keywordID: string) => {
       console.log('Select Keyword: ', keywordID);
       let updatedSelectd = [...selectedKeywords, keywordID];
@@ -194,10 +189,10 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
                               </span>
                               <span className='domKeywords_head_position flex-1 basis-40 grow-0 text-center'>{viewSummary.position}</span>
                               <span className='domKeywords_head_imp flex-1 text-center'>
-                                 {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(viewSummary.impressions)}
+                                 {formattedNum(viewSummary.impressions)}
                               </span>
                               <span className='domKeywords_head_visits flex-1 text-center'>
-                                 {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(viewSummary.visits)}
+                                 {formattedNum(viewSummary.visits)}
                               </span>
                               <span className='domKeywords_head_ctr flex-1 text-center'>
                                  {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(viewSummary.ctr)}%
@@ -214,7 +209,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
                      )}
                      {!isConsoleIntegrated && (
                         <p className=' p-9 pt-[10%] text-center text-gray-500'>
-                        Google Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        Google Search Console has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
                         </p>
                      )}
                   </div>

components/settings/NotificationSettings.tsx

@@ -1,5 +1,7 @@
 import React from 'react';
 import SelectField from '../common/SelectField';
+import SecretField from '../common/SecretField';
+import InputField from '../common/InputField';
 
 type NotificationSettingsProps = {
    settings: SettingsType,
@@ -17,8 +19,8 @@ const NotificationSettings = ({ settings, settingsError, updateSettings }:Notifi
       <div>
          <div className='settings__content styled-scrollbar p-6 text-sm'>
             <div className="settings__section__input mb-5">
-               <label className={labelStyle}>Notification Frequency</label>
                <SelectField
+               label='Notification Frequency'
                   multiple={false}
                   selected={[settings.notification_interval]}
                   options={[
@@ -31,70 +33,61 @@ const NotificationSettings = ({ settings, settingsError, updateSettings }:Notifi
                   updateField={(updated:string[]) => updated[0] && updateSettings('notification_interval', updated[0])}
                   rounded='rounded'
                   maxHeight={48}
-                  minWidth={270}
+                  minWidth={220}
                />
             </div>
             {settings.notification_interval !== 'never' && (
                <>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>Notification Emails</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError?.type === 'no_email' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
-                        value={settings?.notification_email}
-                        placeholder={'test@gmail.com'}
-                        onChange={(event) => updateSettings('notification_email', event.target.value)}
+                     <InputField
+                     label='Notification Emails'
+                     hasError={settingsError?.type === 'no_email'}
+                     value={settings?.notification_email}
+                     placeholder={'test@gmail.com, test2@test.com'}
+                     onChange={(value:string) => updateSettings('notification_email', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Server</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError?.type === 'no_smtp_server' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
-                        value={settings?.smtp_server || ''}
-                        onChange={(event) => updateSettings('smtp_server', event.target.value)}
+                     <InputField
+                     label='SMTP Server'
+                     hasError={settingsError?.type === 'no_smtp_server'}
+                     value={settings?.smtp_server || ''}
+                     placeholder={'test@gmail.com, test2@test.com'}
+                     onChange={(value:string) => updateSettings('smtp_server', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Port</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError && settingsError.type === 'no_smtp_port' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
-                        value={settings?.smtp_port || ''}
-                        onChange={(event) => updateSettings('smtp_port', event.target.value)}
+                     <InputField
+                     label='SMTP Port'
+                     hasError={settingsError?.type === 'no_smtp_port'}
+                     value={settings?.smtp_port || ''}
+                     placeholder={'2234'}
+                     onChange={(value:string) => updateSettings('smtp_port', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Username</label>
-                     <input
-                        className={'w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200'}
-                        type="text"
+                     <InputField
+                        label='SMTP Username'
+                        hasError={settingsError?.type === 'no_smtp_port'}
                         value={settings?.smtp_username || ''}
-                        onChange={(event) => updateSettings('smtp_username', event.target.value)}
+                        onChange={(value:string) => updateSettings('smtp_username', value)}
+                        />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                     <SecretField
+                     label='SMTP Password'
+                     value={settings?.smtp_password || ''}
+                     onChange={(value:string) => updateSettings('smtp_password', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Password</label>
-                     <input
-                        className={'w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200'}
-                        type="text"
-                        value={settings?.smtp_password || ''}
-                        onChange={(event) => updateSettings('smtp_password', event.target.value)}
-                     />
-                  </div>
-                  <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>From Email Address</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError?.type === 'no_smtp_from' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
+                        <InputField
+                        label='From Email Address'
+                        hasError={settingsError?.type === 'no_smtp_from'}
                         value={settings?.notification_email_from || ''}
                         placeholder="no-reply@mydomain.com"
-                        onChange={(event) => updateSettings('notification_email_from', event.target.value)}
-                     />
+                        onChange={(value:string) => updateSettings('notification_email_from', value)}
+                        />
                   </div>
                </>
             )}

components/settings/ScraperSettings.tsx

@@ -2,6 +2,8 @@ import React from 'react';
 import { useClearFailedQueue } from '../../services/settings';
 import Icon from '../common/Icon';
 import SelectField, { SelectionOption } from '../common/SelectField';
+import SecretField from '../common/SecretField';
+import ToggleField from '../common/ToggleField';
 
 type ScraperSettingsProps = {
    settings: SettingsType,
@@ -43,29 +45,25 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
       <div className='settings__content styled-scrollbar p-6 text-sm'>
 
          <div className="settings__section__select mb-5">
-            <label className={labelStyle}>Scraping Method</label>
             <SelectField
+            label='Scraping Method'
             options={scraperOptions}
             selected={[settings.scraper_type || 'none']}
             defaultLabel="Select Scraper"
             updateField={(updatedTime:[string]) => updateSettings('scraper_type', updatedTime[0])}
             multiple={false}
             rounded={'rounded'}
-            minWidth={270}
+            minWidth={220}
             />
          </div>
-         {['scrapingant', 'scrapingrobot', 'serply', 'serpapi', 'spaceSerp'].includes(settings.scraper_type) && (
-            <div className="settings__section__input mr-3">
-               <label className={labelStyle}>Scraper API Key or Token</label>
-               <input
-                  className={`w-full p-2 border border-gray-200 rounded mt-2 mb-3 focus:outline-none  focus:border-blue-200 
-                  ${settingsError?.type === 'no_api_key' ? ' border-red-400 focus:border-red-400' : ''} `}
-                  type="text"
-                  value={settings?.scaping_api || ''}
-                  placeholder={'API Key/Token'}
-                  onChange={(event) => updateSettings('scaping_api', event.target.value)}
-               />
-            </div>
+         {settings.scraper_type !== 'none' && settings.scraper_type !== 'proxy' && (
+            <SecretField
+            label='Scraper API Key or Token'
+            placeholder={'API Key/Token'}
+            value={settings?.scaping_api || ''}
+            hasError={settingsError?.type === 'no_api_key'}
+            onChange={(value:string) => updateSettings('scaping_api', value)}
+            />
          )}
          {settings.scraper_type === 'proxy' && (
             <div className="settings__section__input mb-5">
@@ -82,8 +80,8 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
          )}
          {settings.scraper_type !== 'none' && (
             <div className="settings__section__input mb-5">
-               <label className={labelStyle}>Scraping Frequency</label>
                <SelectField
+                  label='Scraping Frequency'
                   multiple={false}
                   selected={[settings?.scrape_interval || 'daily']}
                   options={scrapingOptions}
@@ -91,14 +89,14 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
                   updateField={(updated:string[]) => updated[0] && updateSettings('scrape_interval', updated[0])}
                   rounded='rounded'
                   maxHeight={48}
-                  minWidth={270}
+                  minWidth={220}
                />
                <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
             </div>
          )}
             <div className="settings__section__input mb-5">
-               <label className={labelStyle}>Delay Between Each keyword Scrape</label>
                <SelectField
+                  label='keyword Scrape Delay'
                   multiple={false}
                   selected={[settings?.scrape_delay || '0']}
                   options={delayOptions}
@@ -106,28 +104,16 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
                   updateField={(updated:string[]) => updated[0] && updateSettings('scrape_delay', updated[0])}
                   rounded='rounded'
                   maxHeight={48}
-                  minWidth={270}
+                  minWidth={220}
                />
                <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
             </div>
             <div className="settings__section__input mb-5">
-               <label className="relative inline-flex items-center cursor-pointer w-full justify-between">
-                  <span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-56">Auto Retry Failed Keyword Scrape</span>
-                  <input
-                  type="checkbox"
-                  value={settings?.scrape_retry ? 'true' : '' }
-                  checked={settings.scrape_retry || false}
-                  className="sr-only peer"
-                  onChange={() => updateSettings('scrape_retry', !settings.scrape_retry)}
-                  />
-                  <div className="relative rounded-3xl w-9 h-5 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
-                  peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
-                  peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
-                  after:absolute after:top-[2px] after:left-[2px] after:bg-white  after:border-gray-300
-                  after:border after:rounded-full after:h-4 after:w-4
-                  after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
-
-               </label>
+               <ToggleField
+               label='Auto Retry Failed Keyword Scrape'
+               value={settings?.scrape_retry ? 'true' : '' }
+               onChange={(val) => updateSettings('scrape_retry', val)}
+               />
             </div>
             {settings?.scrape_retry && (settings.failed_queue?.length || 0) > 0 && (
                <div className="settings__section__input mb-5">

components/settings/SearchConsoleSettings.tsx

@@ -0,0 +1,49 @@
+import React from 'react';
+import ToggleField from '../common/ToggleField';
+import InputField from '../common/InputField';
+
+type SearchConsoleSettingsProps = {
+   settings: SettingsType,
+   settingsError: null | {
+      type: string,
+      msg: string
+   },
+   updateSettings: Function,
+}
+
+const SearchConsoleSettings = ({ settings, settingsError, updateSettings }:SearchConsoleSettingsProps) => {
+   return (
+      <div>
+      <div className='settings__content styled-scrollbar p-6 text-sm'>
+
+         {/* <div className="settings__section__input mb-5">
+            <ToggleField
+            label='Enable Goolge Search Console'
+            value={settings?.scrape_retry ? 'true' : '' }
+            onChange={(val) => updateSettings('scrape_retry', val)}
+            />
+         </div> */}
+         <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+            <InputField
+            label='Search Console Client Email'
+            onChange={(client_email:string) => updateSettings('search_console_client_email', client_email)}
+            value={settings.search_console_client_email}
+            placeholder='myapp@appspot.gserviceaccount.com'
+            />
+         </div>
+         <div className="settings__section__input mb-4 flex flex-col justify-between items-center w-full">
+            <label className='mb-2 font-semibold block text-sm text-gray-700 capitalize w-full'>Search Console Private Key</label>
+            <textarea
+               className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
+               focus:outline-none h-[100px] focus:border-blue-200`}
+               value={settings.search_console_private_key}
+               placeholder={'-----BEGIN PRIVATE KEY-----/ssssaswdkihad....'}
+               onChange={(event) => updateSettings('search_console_private_key', event.target.value)}
+            />
+         </div>
+      </div>
+   </div>
+   );
+};
+
+export default SearchConsoleSettings;

components/settings/Settings.tsx

@@ -1,9 +1,11 @@
 import React, { useEffect, useState } from 'react';
 import { Toaster } from 'react-hot-toast';
-import useUpdateSettings, { useFetchSettings } from '../../services/settings';
+import { useFetchSettings, useUpdateSettings } from '../../services/settings';
 import Icon from '../common/Icon';
 import NotificationSettings from './NotificationSettings';
 import ScraperSettings from './ScraperSettings';
+import useOnKey from '../../hooks/useOnKey';
+import SearchConsoleSettings from './SearchConsoleSettings';
 
 type SettingsProps = {
    closeSettings: Function,
@@ -15,7 +17,7 @@ type SettingsError = {
    msg: string
 }
 
-const defaultSettings = {
+const defaultSettings: SettingsType = {
    scraper_type: 'none',
    scrape_delay: 'none',
    scrape_retry: false,
@@ -26,6 +28,9 @@ const defaultSettings = {
    smtp_username: '',
    smtp_password: '',
    notification_email_from: '',
+   search_console: true,
+   search_console_client_email: '',
+   search_console_private_key: '',
 };
 
 const Settings = ({ closeSettings }:SettingsProps) => {
@@ -34,6 +39,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
    const [settingsError, setSettingsError] = useState<SettingsError|null>(null);
    const { mutate: updateMutate, isLoading: isUpdating } = useUpdateSettings(() => console.log(''));
    const { data: appSettings, isLoading } = useFetchSettings();
+   useOnKey('Escape', closeSettings);
 
    useEffect(() => {
       if (appSettings && appSettings.settings) {
@@ -41,19 +47,6 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       }
    }, [appSettings]);
 
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            console.log(event.key);
-            closeSettings();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeSettings]);
-
    const closeOnBGClick = (e:React.SyntheticEvent) => {
       e.stopPropagation();
       e.nativeEvent.stopImmediatePropagation();
@@ -92,30 +85,38 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       }
    };
 
-   const tabStyle = 'inline-block px-4 py-1 rounded-full mr-3 cursor-pointer text-sm';
+   const tabStyle = `inline-block px-3 py-2 rounded-md  cursor-pointer text-xs lg:text-sm lg:mr-3 lg:px-4 select-none z-10
+   text-gray-600 border border-b-0 relative top-[1px] rounded-b-none`;
+   const tabStyleActive = 'bg-white text-blue-600 border-slate-200';
+
    return (
        <div className="settings fixed w-full h-screen top-0 left-0 z-50" onClick={closeOnBGClick}>
-            <div className="absolute w-full max-w-xs bg-white customShadow top-0 right-0 h-screen" data-loading={isLoading} >
+            <div className="absolute w-full max-w-md bg-white customShadow top-0 right-0 h-screen" data-loading={isLoading} >
                {isLoading && <div className='absolute flex content-center items-center h-full'><Icon type="loading" size={24} /></div>}
-               <div className='settings__header p-6 border-b border-b-slate-200 text-slate-500'>
+               <div className='settings__header px-5 py-4 text-slate-500'>
                   <h3 className=' text-black text-lg font-bold'>Settings</h3>
                   <button
-                  className=' absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
+                  className=' absolute top-2 right-2 p-2 px- text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
                   onClick={() => closeSettings()}>
                      <Icon type='close' size={24} />
                   </button>
                </div>
-               <div className=' px-4 mt-4 '>
+               <div className='border border-slate-200 px-3 py-4 pb-0 border-l-0 border-r-0 bg-[#f8f9ff]'>
                   <ul>
                      <li
-                     className={`${tabStyle} ${currentTab === 'scraper' ? ' bg-blue-50 text-blue-600' : ''}`}
+                     className={`${tabStyle} ${currentTab === 'scraper' ? tabStyleActive : 'border-transparent '}`}
                      onClick={() => setCurrentTab('scraper')}>
-                        Scraper
+                       <Icon type='scraper' /> Scraper
                      </li>
                      <li
-                     className={`${tabStyle} ${currentTab === 'notification' ? ' bg-blue-50 text-blue-600' : ''}`}
+                     className={`${tabStyle} ${currentTab === 'notification' ? tabStyleActive : 'border-transparent'}`}
                      onClick={() => setCurrentTab('notification')}>
-                        Notification
+                        <Icon type='email' /> Notification
+                     </li>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'searchconsole' ? tabStyleActive : 'border-transparent'}`}
+                     onClick={() => setCurrentTab('searchconsole')}>
+                       <Icon type='google' size={14} /> Search Console
                      </li>
                   </ul>
                </div>
@@ -126,6 +127,9 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                {currentTab === 'notification' && settings && (
                   <NotificationSettings settings={settings} updateSettings={updateSettings} settingsError={settingsError} />
                )}
+               {currentTab === 'searchconsole' && settings && (
+                  <SearchConsoleSettings settings={settings} updateSettings={updateSettings} settingsError={settingsError} />
+               )}
                <div className=' border-t-[1px] border-gray-200 p-2 px-3'>
                   <button
                   onClick={() => performUpdate()}

database/config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  production: {
+    username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER,
+    password: process.env.PASSWORD,
+    database: 'sequelize',
+    host: 'database',
+    port: 3306,
+    dialect: 'sqlite',
+    storage: './data/database.sqlite',
+    dialectOptions: {
+      bigNumberStrings: true,
+    },
+  },
+};

database/migrations/1707068556345-add-new-keyword-fields.js

@@ -0,0 +1,45 @@
+// Migration: Adds city, latlong and settings keyword to keyword table.
+
+// CLI Migration
+module.exports = {
+   up: async (queryInterface, Sequelize) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (!keywordTableDefinition.city) {
+                  await queryInterface.addColumn('keyword', 'city', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+               if (!keywordTableDefinition.latlong) {
+                  await queryInterface.addColumn('keyword', 'latlong', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+               if (!keywordTableDefinition.settings) {
+                  await queryInterface.addColumn('keyword', 'settings', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+   down: (queryInterface) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (keywordTableDefinition.city) {
+                  await queryInterface.removeColumn('keyword', 'city', { transaction: t });
+               }
+               if (keywordTableDefinition.latlong) {
+                  await queryInterface.removeColumn('keyword', 'latlong', { transaction: t });
+               }
+               if (keywordTableDefinition.latlong) {
+                  await queryInterface.removeColumn('keyword', 'settings', { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+ };

database/migrations/1707233039698-add-domain-searchconsole-field.js

@@ -0,0 +1,29 @@
+// Migration: Adds search_console field to domain table to assign search console property type, url and api.
+
+// CLI Migration
+module.exports = {
+   up: (queryInterface, Sequelize) => {
+     return queryInterface.sequelize.transaction(async (t) => {
+      try {
+         const domainTableDefinition = await queryInterface.describeTable('domain');
+         if (domainTableDefinition && !domainTableDefinition.search_console) {
+            await queryInterface.addColumn('domain', 'search_console', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+         }
+      } catch (error) {
+         console.log('error :', error);
+      }
+     });
+   },
+   down: (queryInterface) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const domainTableDefinition = await queryInterface.describeTable('domain');
+            if (domainTableDefinition && domainTableDefinition.search_console) {
+               await queryInterface.removeColumn('domain', 'search_console', { transaction: t });
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+ };

database/models/domain.ts

@@ -38,6 +38,9 @@ class Domain extends Model {
 
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
    notification_emails!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true })
+   search_console!: string;
 }
 
 export default Domain;

database/models/keyword.ts

@@ -19,7 +19,13 @@ class Keyword extends Model {
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: 'US' })
    country!: string;
 
-   @Column({ type: DataType.STRING, allowNull: false })
+   @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
+   city!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
+   latlong!: string;
+
+   @Column({ type: DataType.STRING, allowNull: false, defaultValue: '{}' })
    domain!: string;
 
    // @ForeignKey(() => Domain)
@@ -58,6 +64,9 @@ class Keyword extends Model {
 
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: 'false' })
    lastUpdateError!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true })
+   settings!: string;
 }
 
 export default Keyword;

entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+npx sequelize-cli db:migrate --env production
+exec "$@"

hooks/useIsMobile.tsx

@@ -0,0 +1,12 @@
+import { useEffect, useState } from 'react';
+
+const useIsMobile = () => {
+   const [isMobile, setIsMobile] = useState<boolean>(false);
+   useEffect(() => {
+      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
+   }, []);
+
+   return [isMobile];
+};
+
+export default useIsMobile;

hooks/useOnKey.tsx

@@ -0,0 +1,17 @@
+import { useEffect } from 'react';
+
+const useOnKey = (key:string, onPress: Function) => {
+   useEffect(() => {
+      const closeModalonEsc = (event:KeyboardEvent) => {
+         if (event.key === key) {
+            onPress();
+         }
+      };
+      window.addEventListener('keydown', closeModalonEsc, false);
+      return () => {
+         window.removeEventListener('keydown', closeModalonEsc, false);
+      };
+   }, [key, onPress]);
+};
+
+export default useOnKey;

hooks/useWindowResize.tsx

@@ -0,0 +1,13 @@
+import { useEffect } from 'react';
+
+const useWindowResize = (onResize: () => void) => {
+   useEffect(() => {
+      onResize();
+      window.addEventListener('resize', onResize);
+      return () => {
+          window.removeEventListener('resize', onResize);
+      };
+   }, [onResize]);
+};
+
+export default useWindowResize;

jest.setup.js

@@ -1,11 +1,26 @@
 // eslint-disable-next-line no-unused-vars
 import 'isomorphic-fetch';
 import './styles/globals.css';
+import '@testing-library/jest-dom';
+import { enableFetchMocks } from 'jest-fetch-mock';
 // Optional: configure or set up a testing framework before each test.
 // If you delete this file, remove `setupFilesAfterEnv` from `jest.config.js`
 
 // Used for __tests__/testing-library.js
 // Learn more: https://github.com/testing-library/jest-dom
-import '@testing-library/jest-dom';
+
+window.matchMedia = (query) => ({
+   matches: false,
+   media: query,
+   onchange: null,
+   addListener: jest.fn(), // deprecated
+   removeListener: jest.fn(), // deprecated
+   addEventListener: jest.fn(),
+   removeEventListener: jest.fn(),
+   dispatchEvent: jest.fn(),
+});
 
 global.ResizeObserver = require('resize-observer-polyfill');
+
+// Enable Fetch Mocking
+enableFetchMocks();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "0.3.1",
+  "version": "1.0.2",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -13,6 +13,8 @@
     "test": "jest --watch --verbose",
     "test:ci": "jest --ci",
     "test:cv": "jest --coverage --coverageDirectory='coverage'",
+    "db:migrate": "sequelize-cli db:migrate --env production",
+    "db:revert": "sequelize-cli db:migrate:undo --env production",
     "release": "standard-version"
   },
   "dependencies": {
@@ -33,7 +35,7 @@
     "jsonwebtoken": "^9.0.2",
     "msw": "^0.49.0",
     "next": "^12.3.4",
-    "nodemailer": "^6.8.0",
+    "nodemailer": "^6.9.9",
     "react": "18.2.0",
     "react-chartjs-2": "^4.3.1",
     "react-dom": "18.2.0",
@@ -44,8 +46,9 @@
     "react-window": "^1.8.8",
     "reflect-metadata": "^0.1.13",
     "sequelize": "^6.34.0",
-    "sequelize-typescript": "^2.1.5",
-    "sqlite3": "^5.1.6"
+    "sequelize-typescript": "^2.1.6",
+    "sqlite3": "^5.1.6",
+    "umzug": "^3.6.1"
   },
   "devDependencies": {
     "@testing-library/jest-dom": "^6.1.4",
@@ -53,6 +56,7 @@
     "@types/cookies": "^0.7.7",
     "@types/cryptr": "^4.0.1",
     "@types/isomorphic-fetch": "^0.0.36",
+    "@types/jest": "^29.5.8",
     "@types/jsonwebtoken": "^8.5.9",
     "@types/node": "18.11.0",
     "@types/nodemailer": "^6.4.6",
@@ -66,10 +70,13 @@
     "eslint-config-next": "12.3.1",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
+    "jest-fetch-mock": "^3.0.3",
+    "next-router-mock": "^0.9.10",
     "postcss": "^8.4.31",
     "prettier": "^2.7.1",
     "resize-observer-polyfill": "^1.5.1",
     "sass": "^1.55.0",
+    "sequelize-cli": "^6.6.2",
     "standard-version": "^9.5.0",
     "stylelint-config-standard": "^29.0.0",
     "tailwindcss": "^3.1.8",

pages/api/dbmigrate.ts

@@ -0,0 +1,53 @@
+import { Sequelize } from 'sequelize';
+import { Umzug, SequelizeStorage } from 'umzug';
+import type { NextApiRequest, NextApiResponse } from 'next';
+import db from '../../database/database';
+import verifyUser from '../../utils/verifyUser';
+
+type MigrationGetResponse = {
+   hasMigrations: boolean,
+}
+
+type MigrationPostResponse = {
+   migrated: boolean,
+   erroor?: string
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized === 'authorized' && req.method === 'GET') {
+      await db.sync();
+      return getMigrationStatus(req, res);
+   }
+   if (authorized === 'authorized' && req.method === 'POST') {
+      return migrateDatabase(req, res);
+   }
+   return res.status(401).json({ error: authorized });
+}
+
+const getMigrationStatus = async (req: NextApiRequest, res: NextApiResponse<MigrationGetResponse>) => {
+   const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false });
+   const umzug = new Umzug({
+      migrations: { glob: 'database/migrations/*.js' },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: undefined,
+   });
+   const migrations = await umzug.pending();
+   // console.log('migrations :', migrations);
+   // const migrationsExceuted = await umzug.executed();
+   return res.status(200).json({ hasMigrations: migrations.length > 0 });
+};
+
+const migrateDatabase = async (req: NextApiRequest, res: NextApiResponse<MigrationPostResponse>) => {
+   const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false });
+   const umzug = new Umzug({
+      migrations: { glob: 'database/migrations/*.js' },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: undefined,
+   });
+   const migrations = await umzug.up();
+   console.log('[Updated] migrations :', migrations);
+   return res.status(200).json({ migrated: true });
+};

pages/api/domain.ts

@@ -0,0 +1,48 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import Cryptr from 'cryptr';
+import db from '../../database/database';
+import Domain from '../../database/models/domain';
+import verifyUser from '../../utils/verifyUser';
+
+type DomainGetResponse = {
+   domain?: DomainType | null
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized === 'authorized' && req.method === 'GET') {
+      await db.sync();
+      return getDomain(req, res);
+   }
+   return res.status(401).json({ error: authorized });
+}
+
+const getDomain = async (req: NextApiRequest, res: NextApiResponse<DomainGetResponse>) => {
+   if (!req.query.domain && typeof req.query.domain !== 'string') {
+       return res.status(400).json({ error: 'Domain Name is Required!' });
+   }
+
+   try {
+      const query = { domain: req.query.domain as string };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const parsedDomain = foundDomain?.get({ plain: true }) || false;
+
+      if (parsedDomain && parsedDomain.search_console) {
+         try {
+            const cryptr = new Cryptr(process.env.SECRET as string);
+            const scData = JSON.parse(parsedDomain.search_console);
+            scData.client_email = scData.client_email ? cryptr.decrypt(scData.client_email) : '';
+            scData.private_key = scData.private_key ? cryptr.decrypt(scData.private_key) : '';
+            parsedDomain.search_console = JSON.stringify(scData);
+         } catch (error) {
+            console.log('[Error] Parsing Search Console Keys.');
+         }
+      }
+
+      return res.status(200).json({ domain: parsedDomain });
+   } catch (error) {
+      console.log('[ERROR] Getting Domain: ', error);
+      return res.status(400).json({ error: 'Error Loading Domain' });
+   }
+};

pages/api/domains.ts

@@ -1,9 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
+import Cryptr from 'cryptr';
 import db from '../../database/database';
 import Domain from '../../database/models/domain';
 import Keyword from '../../database/models/keyword';
 import getdomainStats from '../../utils/domains';
 import verifyUser from '../../utils/verifyUser';
+import { checkSerchConsoleIntegration, removeLocalSCData } from '../../utils/searchConsole';
 
 type DomainsGetRes = {
    domains: DomainType[]
@@ -11,13 +13,14 @@ type DomainsGetRes = {
 }
 
 type DomainsAddResponse = {
-   domain: Domain|null,
+   domains: DomainType[]|null,
    error?: string|null,
 }
 
 type DomainsDeleteRes = {
    domainRemoved: number,
    keywordsRemoved: number,
+   SCDataRemoved: boolean,
    error?: string|null,
 }
 
@@ -51,7 +54,13 @@ export const getDomains = async (req: NextApiRequest, res: NextApiResponse<Domai
    const withStats = !!req?.query?.withstats;
    try {
       const allDomains: Domain[] = await Domain.findAll();
-      const formattedDomains: DomainType[] = allDomains.map((el) => el.get({ plain: true }));
+      const formattedDomains: DomainType[] = allDomains.map((el) => {
+         const domainItem:DomainType = el.get({ plain: true });
+         const scData = domainItem?.search_console ? JSON.parse(domainItem.search_console) : {};
+         const { client_email, private_key } = scData;
+         const searchConsoleData = scData ? { ...scData, client_email: client_email ? 'true' : '', private_key: private_key ? 'true' : '' } : {};
+         return { ...domainItem, search_console: JSON.stringify(searchConsoleData) };
+      });
       const theDomains: DomainType[] = withStats ? await getdomainStats(formattedDomains) : allDomains;
       return res.status(200).json({ domains: theDomains });
    } catch (error) {
@@ -59,41 +68,45 @@ export const getDomains = async (req: NextApiRequest, res: NextApiResponse<Domai
    }
 };
 
-export const addDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsAddResponse>) => {
-   if (!req.body.domain) {
-      return res.status(400).json({ domain: null, error: 'Error Adding Domain.' });
-   }
-   const { domain } = req.body || {};
-   const domainData = {
-      domain: domain.trim(),
-      slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-'),
-      lastUpdated: new Date().toJSON(),
-      added: new Date().toJSON(),
-   };
+const addDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsAddResponse>) => {
+   const { domains } = req.body;
+   if (domains && Array.isArray(domains) && domains.length > 0) {
+      const domainsToAdd: any = [];
 
-   try {
-      const addedDomain = await Domain.create(domainData);
-      return res.status(201).json({ domain: addedDomain });
-   } catch (error) {
-      return res.status(400).json({ domain: null, error: 'Error Adding Domain.' });
+      domains.forEach((domain: string) => {
+         domainsToAdd.push({
+            domain: domain.trim(),
+            slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-').replaceAll('/', '-'),
+            lastUpdated: new Date().toJSON(),
+            added: new Date().toJSON(),
+         });
+      });
+      try {
+         const newDomains:Domain[] = await Domain.bulkCreate(domainsToAdd);
+         const formattedDomains = newDomains.map((el) => el.get({ plain: true }));
+         return res.status(201).json({ domains: formattedDomains });
+      } catch (error) {
+         console.log('[ERROR] Adding New Domain ', error);
+         return res.status(400).json({ domains: [], error: 'Error Adding Domain.' });
+      }
+   } else {
+      return res.status(400).json({ domains: [], error: 'Necessary data missing.' });
    }
 };
 
 export const deleteDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsDeleteRes>) => {
    if (!req.query.domain && typeof req.query.domain !== 'string') {
-      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, error: 'Domain is Required!' });
+      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, SCDataRemoved: false, error: 'Domain is Required!' });
    }
    try {
       const { domain } = req.query || {};
       const removedDomCount: number = await Domain.destroy({ where: { domain } });
       const removedKeywordCount: number = await Keyword.destroy({ where: { domain } });
-      return res.status(200).json({
-            domainRemoved: removedDomCount,
-            keywordsRemoved: removedKeywordCount,
-         });
+      const SCDataRemoved = await removeLocalSCData(domain as string);
+      return res.status(200).json({ domainRemoved: removedDomCount, keywordsRemoved: removedKeywordCount, SCDataRemoved });
    } catch (error) {
       console.log('[ERROR] Deleting Domain: ', req.query.domain, error);
-      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, error: 'Error Deleting Domain' });
+      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, SCDataRemoved: false, error: 'Error Deleting Domain' });
    }
 };
 
@@ -102,17 +115,28 @@ export const updateDomain = async (req: NextApiRequest, res: NextApiResponse<Dom
       return res.status(400).json({ domain: null, error: 'Domain is Required!' });
    }
    const { domain } = req.query || {};
-   const { notification_interval, notification_emails } = req.body;
+   const { notification_interval, notification_emails, search_console } = req.body as DomainSettings;
 
    try {
       const domainToUpdate: Domain|null = await Domain.findOne({ where: { domain } });
+      // Validate Search Console API Data
+      if (domainToUpdate && search_console?.client_email && search_console?.private_key) {
+         const theDomainObj = domainToUpdate.get({ plain: true });
+         const isSearchConsoleAPIValid = await checkSerchConsoleIntegration({ ...theDomainObj, search_console: JSON.stringify(search_console) });
+         if (!isSearchConsoleAPIValid.isValid) {
+            return res.status(400).json({ domain: null, error: isSearchConsoleAPIValid.error });
+         }
+         const cryptr = new Cryptr(process.env.SECRET as string);
+         search_console.client_email = search_console.client_email ? cryptr.encrypt(search_console.client_email.trim()) : '';
+         search_console.private_key = search_console.private_key ? cryptr.encrypt(search_console.private_key.trim()) : '';
+      }
       if (domainToUpdate) {
-         domainToUpdate.set({ notification_interval, notification_emails });
+         domainToUpdate.set({ notification_interval, notification_emails, search_console: JSON.stringify(search_console) });
          await domainToUpdate.save();
       }
       return res.status(200).json({ domain: domainToUpdate });
    } catch (error) {
       console.log('[ERROR] Updating Domain: ', req.query.domain, error);
-      return res.status(400).json({ domain: null, error: 'Error Updating Domain' });
+      return res.status(400).json({ domain: null, error: 'Error Updating Domain. An Unknown Error Occured.' });
    }
 };

pages/api/insight.ts

@@ -1,8 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import db from '../../database/database';
 import { getCountryInsight, getKeywordsInsight, getPagesInsight } from '../../utils/insight';
-import { fetchDomainSCData, readLocalSCData } from '../../utils/searchConsole';
+import { fetchDomainSCData, getSearchConsoleApiInfo, readLocalSCData } from '../../utils/searchConsole';
 import verifyUser from '../../utils/verifyUser';
+import Domain from '../../database/models/domain';
 
 type SCInsightRes = {
    data: InsightDataType | null,
@@ -23,9 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
 const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiResponse<SCInsightRes>) => {
    if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' });
-   if (!!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) === false) {
-      return res.status(200).json({ data: null, error: 'Google Search Console Not Integrated' });
-   }
    const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
    const getInsightFromSCData = (localSCData: SCDomainDataType): InsightDataType => {
       const { stats = [] } = localSCData;
@@ -37,17 +35,26 @@ const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiRe
 
    // First try and read the  Local SC Domain Data file.
    const localSCData = await readLocalSCData(domainname);
-   const oldFetchedDate = localSCData.lastFetched;
-   const fetchTimeDiff = new Date().getTime() - (oldFetchedDate ? new Date(oldFetchedDate as string).getTime() : 0);
 
-   if (localSCData && localSCData.stats && localSCData.stats.length && fetchTimeDiff <= 86400000) {
-      const response = getInsightFromSCData(localSCData);
-      return res.status(200).json({ data: response });
+   if (localSCData) {
+      const oldFetchedDate = localSCData.lastFetched;
+      const fetchTimeDiff = new Date().getTime() - (oldFetchedDate ? new Date(oldFetchedDate as string).getTime() : 0);
+      if (localSCData.stats && localSCData.stats.length && fetchTimeDiff <= 86400000) {
+         const response = getInsightFromSCData(localSCData);
+         return res.status(200).json({ data: response });
+      }
    }
 
    // If the Local SC Domain Data file does not exist, fetch from Googel Search Console.
    try {
-      const scData = await fetchDomainSCData(domainname);
+      const query = { domain: domainname };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
+      const scDomainAPI = await getSearchConsoleApiInfo(domainObj);
+      if (!(scDomainAPI.client_email && scDomainAPI.private_key)) {
+         return res.status(200).json({ data: null, error: 'Google Search Console is not Integrated.' });
+      }
+      const scData = await fetchDomainSCData(domainObj, scDomainAPI);
       const response = getInsightFromSCData(scData);
       return res.status(200).json({ data: response });
    } catch (error) {

pages/api/keywords.ts

@@ -45,7 +45,7 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
    if (!req.query.domain && typeof req.query.domain !== 'string') {
       return res.status(400).json({ error: 'Domain is Required!' });
    }
-   const domain = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
+   const domain = (req.query.domain as string);
    const integratedSC = process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL;
    const domainSCData = integratedSC ? await readLocalSCData(domain) : false;
 
@@ -79,13 +79,14 @@ const addKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
       const keywordsToAdd: any = []; // QuickFIX for bug: https://github.com/sequelize/sequelize-typescript/issues/936
 
       keywords.forEach((kwrd: KeywordAddPayload) => {
-         const { keyword, device, country, domain, tags } = kwrd;
+         const { keyword, device, country, domain, tags, city } = kwrd;
          const tagsArray = tags ? tags.split(',').map((item:string) => item.trim()) : [];
          const newKeyword = {
             keyword,
             device,
             domain,
             country,
+            city,
             position: 0,
             updating: true,
             history: JSON.stringify({}),

pages/api/searchconsole.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import db from '../../database/database';
 import Domain from '../../database/models/domain';
-import { fetchDomainSCData, readLocalSCData } from '../../utils/searchConsole';
+import { fetchDomainSCData, getSearchConsoleApiInfo, readLocalSCData } from '../../utils/searchConsole';
 import verifyUser from '../../utils/verifyUser';
 
 type searchConsoleRes = {
@@ -31,18 +31,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
 const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse<searchConsoleRes>) => {
    if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' });
-   if (!!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) === false) {
-      return res.status(200).json({ data: null, error: 'Google Search Console Not Integrated' });
-   }
    const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
    const localSCData = await readLocalSCData(domainname);
-   console.log(localSCData && localSCData.thirtyDays && localSCData.thirtyDays.length);
-
    if (localSCData && localSCData.thirtyDays && localSCData.thirtyDays.length) {
       return res.status(200).json({ data: localSCData });
    }
    try {
-      const scData = await fetchDomainSCData(domainname);
+      const query = { domain: domainname };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
+      const scDomainAPI = await getSearchConsoleApiInfo(domainObj);
+      if (!(scDomainAPI.client_email && scDomainAPI.private_key)) {
+         return res.status(200).json({ data: null, error: 'Google Search Console is not Integrated.' });
+      }
+      const scData = await fetchDomainSCData(domainObj, scDomainAPI);
       return res.status(200).json({ data: scData });
    } catch (error) {
       console.log('[ERROR] Getting Search Console Data for: ', domainname, error);
@@ -53,9 +55,9 @@ const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiRespo
 const cronRefreshSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse<searchConsoleCRONRes>) => {
    try {
       const allDomainsRaw = await Domain.findAll();
-      const Domains: Domain[] = allDomainsRaw.map((el) => el.get({ plain: true }));
+      const Domains: DomainType[] = allDomainsRaw.map((el) => el.get({ plain: true }));
       for (const domain of Domains) {
-         await fetchDomainSCData(domain.domain);
+         await fetchDomainSCData(domain);
       }
       return res.status(200).json({ status: 'completed' });
    } catch (error) {

pages/api/settings.ts

@@ -42,9 +42,11 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
    }
    try {
       const cryptr = new Cryptr(process.env.SECRET as string);
-      const scaping_api = settings.scaping_api ? cryptr.encrypt(settings.scaping_api) : '';
-      const smtp_password = settings.smtp_password ? cryptr.encrypt(settings.smtp_password) : '';
-      const securedSettings = { ...settings, scaping_api, smtp_password };
+      const scaping_api = settings.scaping_api ? cryptr.encrypt(settings.scaping_api.trim()) : '';
+      const smtp_password = settings.smtp_password ? cryptr.encrypt(settings.smtp_password.trim()) : '';
+      const search_console_client_email = settings.search_console_client_email ? cryptr.encrypt(settings.search_console_client_email.trim()) : '';
+      const search_console_private_key = settings.search_console_private_key ? cryptr.encrypt(settings.search_console_private_key.trim()) : '';
+      const securedSettings = { ...settings, scaping_api, smtp_password, search_console_client_email, search_console_private_key };
 
       await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify(securedSettings), { encoding: 'utf-8' });
       return res.status(200).json({ settings });
@@ -55,6 +57,7 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
 };
 
 export const getAppSettings = async () : Promise<SettingsType> => {
+   const screenshotAPIKey = process.env.SCREENSHOT_API || '69408-serpbear';
    try {
       const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
       const failedQueueRaw = await readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' });
@@ -66,13 +69,19 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          const cryptr = new Cryptr(process.env.SECRET as string);
          const scaping_api = settings.scaping_api ? cryptr.decrypt(settings.scaping_api) : '';
          const smtp_password = settings.smtp_password ? cryptr.decrypt(settings.smtp_password) : '';
+         const search_console_client_email = settings.search_console_client_email ? cryptr.decrypt(settings.search_console_client_email) : '';
+         const search_console_private_key = settings.search_console_private_key ? cryptr.decrypt(settings.search_console_private_key) : '';
          decryptedSettings = {
             ...settings,
             scaping_api,
             smtp_password,
-            search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL),
-            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
+            search_console_client_email,
+            search_console_private_key,
+            search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL)
+            || !!(search_console_client_email && search_console_private_key),
+            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id, allowsCity: !!scraper.allowsCity })),
             failed_queue: failedQueue,
+            screenshot_key: screenshotAPIKey,
          };
       } catch (error) {
          console.log('Error Decrypting Settings API Keys!');
@@ -81,7 +90,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
       return decryptedSettings;
    } catch (error) {
       console.log('[ERROR] Getting App Settings. ', error);
-      const settings = {
+      const settings: SettingsType = {
          scraper_type: 'none',
          notification_interval: 'never',
          notification_email: '',
@@ -91,6 +100,10 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          smtp_username: '',
          smtp_password: '',
          scrape_retry: false,
+         screenshot_key: screenshotAPIKey,
+         search_console: true,
+         search_console_client_email: '',
+         search_console_private_key: '',
       };
       const otherSettings = {
          available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),

pages/domain/[slug]/index.tsx

@@ -1,9 +1,7 @@
-import React, { useEffect, useMemo, useState } from 'react';
+import React, { useMemo, useState } from 'react';
 import type { NextPage } from 'next';
 import Head from 'next/head';
 import { useRouter } from 'next/router';
-// import { useQuery } from 'react-query';
-// import toast from 'react-hot-toast';
 import { CSSTransition } from 'react-transition-group';
 import Sidebar from '../../../components/common/Sidebar';
 import TopBar from '../../../components/common/TopBar';
@@ -11,47 +9,46 @@ import DomainHeader from '../../../components/domains/DomainHeader';
 import KeywordsTable from '../../../components/keywords/KeywordsTable';
 import AddDomain from '../../../components/domains/AddDomain';
 import DomainSettings from '../../../components/domains/DomainSettings';
-import exportCSV from '../../../utils/exportcsv';
+import exportCSV from '../../../utils/client/exportcsv';
 import Settings from '../../../components/settings/Settings';
 import { useFetchDomains } from '../../../services/domains';
 import { useFetchKeywords } from '../../../services/keywords';
 import { useFetchSettings } from '../../../services/settings';
+import AddKeywords from '../../../components/keywords/AddKeywords';
 
 const SingleDomain: NextPage = () => {
    const router = useRouter();
-   const [noScrapprtError, setNoScrapprtError] = useState(false);
    const [showAddKeywords, setShowAddKeywords] = useState(false);
    const [showAddDomain, setShowAddDomain] = useState(false);
    const [showDomainSettings, setShowDomainSettings] = useState(false);
    const [showSettings, setShowSettings] = useState(false);
    const [keywordSPollInterval, setKeywordSPollInterval] = useState<undefined|number>(undefined);
-   const { data: appSettings } = useFetchSettings();
+   const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings();
    const { data: domainsData } = useFetchDomains(router);
-   const { keywordsData, keywordsLoading } = useFetchKeywords(router, setKeywordSPollInterval, keywordSPollInterval);
-
-   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
-   const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
+   const appSettings: SettingsType = appSettingsData?.settings || {};
+   const { scraper_type = '', available_scapers = [] } = appSettings;
+   const activeScraper = useMemo(() => available_scapers.find((scraper) => scraper.value === scraper_type), [scraper_type, available_scapers]);
 
    const activDomain: DomainType|null = useMemo(() => {
       let active:DomainType|null = null;
       if (domainsData?.domains && router.query?.slug) {
-         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug);
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
       }
       return active;
    }, [router.query.slug, domainsData]);
 
-   useEffect(() => {
-      // console.log('appSettings.settings: ', appSettings && appSettings.settings);
-      if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) {
-         setNoScrapprtError(true);
-      }
-   }, [appSettings]);
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
 
-   // console.log('Domains Data:', router, activDomain, theKeywords);
+   const { keywordsData, keywordsLoading } = useFetchKeywords(router, activDomain?.domain || '', setKeywordSPollInterval, keywordSPollInterval);
+   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
+   const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
 
    return (
       <div className="Domain ">
-         {noScrapprtError && (
+         {((!scraper_type || (scraper_type === 'none')) && !isAppSettingsLoading) && (
                <div className=' p-3 bg-red-600 text-white text-sm text-center'>
                   A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
                </div>
@@ -80,13 +77,13 @@ const SingleDomain: NextPage = () => {
                keywords={theKeywords}
                showAddModal={showAddKeywords}
                setShowAddModal={setShowAddKeywords}
-               isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
+               isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) || domainHasScAPI }
                />
             </div>
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
 
          <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
@@ -98,6 +95,15 @@ const SingleDomain: NextPage = () => {
          <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
              <Settings closeSettings={() => setShowSettings(false)} />
          </CSSTransition>
+         <CSSTransition in={showAddKeywords} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddKeywords
+               domain={activDomain?.domain || ''}
+               scraperName={activeScraper?.label || ''}
+               keywords={theKeywords}
+               allowsCity={!!activeScraper?.allowsCity}
+               closeModal={() => setShowAddKeywords(false)}
+               />
+         </CSSTransition>
       </div>
    );
 };

pages/domain/console/[slug]/index.tsx

@@ -10,7 +10,7 @@ import TopBar from '../../../../components/common/TopBar';
 import DomainHeader from '../../../../components/domains/DomainHeader';
 import AddDomain from '../../../../components/domains/AddDomain';
 import DomainSettings from '../../../../components/domains/DomainSettings';
-import exportCSV from '../../../../utils/exportcsv';
+import exportCSV from '../../../../utils/client/exportcsv';
 import Settings from '../../../../components/settings/Settings';
 import { useFetchDomains } from '../../../../services/domains';
 import { useFetchSCKeywords } from '../../../../services/searchConsole';
@@ -34,11 +34,16 @@ const DiscoverPage: NextPage = () => {
    const activDomain: DomainType|null = useMemo(() => {
       let active:DomainType|null = null;
       if (domainsData?.domains && router.query?.slug) {
-         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug);
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
       }
       return active;
    }, [router.query.slug, domainsData]);
 
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
+
    return (
       <div className="Domain ">
          {activDomain && activDomain.domain
@@ -65,13 +70,13 @@ const DiscoverPage: NextPage = () => {
                isLoading={keywordsLoading || isFetching}
                domain={activDomain}
                keywords={theKeywords}
-               isConsoleIntegrated={scConnected}
+               isConsoleIntegrated={scConnected || domainHasScAPI}
                />
             </div>
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
 
          <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>

pages/domain/insight/[slug]/index.tsx

@@ -10,7 +10,7 @@ import TopBar from '../../../../components/common/TopBar';
 import DomainHeader from '../../../../components/domains/DomainHeader';
 import AddDomain from '../../../../components/domains/AddDomain';
 import DomainSettings from '../../../../components/domains/DomainSettings';
-import exportCSV from '../../../../utils/exportcsv';
+import exportCSV from '../../../../utils/client/exportcsv';
 import Settings from '../../../../components/settings/Settings';
 import { useFetchDomains } from '../../../../services/domains';
 import { useFetchSCInsight } from '../../../../services/searchConsole';
@@ -34,11 +34,16 @@ const InsightPage: NextPage = () => {
    const activDomain: DomainType|null = useMemo(() => {
       let active:DomainType|null = null;
       if (domainsData?.domains && router.query?.slug) {
-         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug);
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
       }
       return active;
    }, [router.query.slug, domainsData]);
 
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
+
    return (
       <div className="Domain ">
          {activDomain && activDomain.domain
@@ -65,13 +70,13 @@ const InsightPage: NextPage = () => {
                isLoading={false}
                domain={activDomain}
                insight={theInsight}
-               isConsoleIntegrated={scConnected}
+               isConsoleIntegrated={scConnected || domainHasScAPI}
                />
             </div>
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
 
          <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>

pages/domains/index.tsx

@@ -1,69 +1,92 @@
-import React, { useEffect, useState } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
 import type { NextPage } from 'next';
 import Head from 'next/head';
 import { useRouter } from 'next/router';
 import { CSSTransition } from 'react-transition-group';
+import toast, { Toaster } from 'react-hot-toast';
 import TopBar from '../../components/common/TopBar';
 import AddDomain from '../../components/domains/AddDomain';
 import Settings from '../../components/settings/Settings';
-import { useFetchSettings } from '../../services/settings';
-import { useFetchDomains } from '../../services/domains';
+import { useCheckMigrationStatus, useFetchSettings } from '../../services/settings';
+import { fetchDomainScreenshot, useFetchDomains } from '../../services/domains';
 import DomainItem from '../../components/domains/DomainItem';
 import Icon from '../../components/common/Icon';
 
 type thumbImages = { [domain:string] : string }
 
-const SingleDomain: NextPage = () => {
+const Domains: NextPage = () => {
    const router = useRouter();
-   const [noScrapprtError, setNoScrapprtError] = useState(false);
+   // const [noScrapprtError, setNoScrapprtError] = useState(false);
    const [showSettings, setShowSettings] = useState(false);
    const [showAddDomain, setShowAddDomain] = useState(false);
    const [domainThumbs, setDomainThumbs] = useState<thumbImages>({});
-   const { data: appSettings } = useFetchSettings();
+   const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings();
    const { data: domainsData, isLoading } = useFetchDomains(router, true);
+   const { data: migrationStatus } = useCheckMigrationStatus();
+   // const { mutate: updateDatabaseMutate, isLoading: isUpdatingDB } = useMigrateDatabase((res:Object) => { window.location.reload(); });
+
+   const appSettings:SettingsType = appSettingsData?.settings || {};
+   const { scraper_type = '' } = appSettings;
+
+   const totalKeywords = useMemo(() => {
+      let keywords = 0;
+      if (domainsData?.domains) {
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            keywords += domain?.keywordCount || 0;
+         });
+      }
+      return keywords;
+   }, [domainsData]);
+
+   const domainSCAPiObj = useMemo(() => {
+      const domainsSCAPI:{ [ID:string] : boolean } = {};
+      if (domainsData?.domains) {
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            const doaminSc = domain?.search_console ? JSON.parse(domain.search_console) : {};
+            domainsSCAPI[domain.ID] = doaminSc.client_email && doaminSc.private_key;
+         });
+      }
+      return domainsSCAPI;
+   }, [domainsData]);
 
    useEffect(() => {
-      // console.log('Domains Data: ', domainsData);
-      if (domainsData?.domains && domainsData.domains.length > 0) {
-         const domainThumbsRaw = localStorage.getItem('domainThumbs');
-         const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
+      if (domainsData?.domains && domainsData.domains.length > 0 && appSettings.screenshot_key) {
          domainsData.domains.forEach(async (domain:DomainType) => {
             if (domain.domain) {
-               if (!domThumbs[domain.domain]) {
-                  const domainImageBlob = await fetch(`https://image.thum.io/get/auth/66909-serpbear/maxAge/96/width/200/https://${domain.domain}`).then((res) => res.blob());
-                  if (domainImageBlob) {
-                     const reader = new FileReader();
-                     await new Promise((resolve, reject) => {
-                       reader.onload = resolve;
-                       reader.onerror = reject;
-                       reader.readAsDataURL(domainImageBlob);
-                     });
-                     const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
-                     localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain.domain]: imageBase }));
-                     setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: imageBase }));
-                  }
-               } else {
-                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domThumbs[domain.domain] }));
+               const domainThumb = await fetchDomainScreenshot(domain.domain, appSettings.screenshot_key || '');
+               if (domainThumb) {
+                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domainThumb }));
                }
             }
          });
       }
-   }, [domainsData]);
+   }, [domainsData, appSettings.screenshot_key]);
 
-   useEffect(() => {
-      // console.log('appSettings.settings: ', appSettings && appSettings.settings);
-      if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) {
-         setNoScrapprtError(true);
+   const manuallyUpdateThumb = async (domain: string) => {
+      if (domain && appSettings.screenshot_key) {
+         const domainThumb = await fetchDomainScreenshot(domain, appSettings.screenshot_key, true);
+         if (domainThumb) {
+            toast(`${domain} Screenshot Updated Successfully!`, { icon: '✔️' });
+            setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain]: domainThumb }));
+         } else {
+            toast(`Failed to Fetch ${domain} Screenshot!`, { icon: '⚠️' });
+         }
       }
-   }, [appSettings]);
+   };
 
    return (
-      <div className="Domain flex flex-col min-h-screen">
-         {noScrapprtError && (
+      <div data-testid="domains" className="Domain flex flex-col min-h-screen">
+         {((!scraper_type || (scraper_type === 'none')) && !isAppSettingsLoading) && (
                <div className=' p-3 bg-red-600 text-white text-sm text-center'>
                   A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
                </div>
          )}
+         {migrationStatus?.hasMigrations && (
+               <div className=' p-3 bg-black text-white text-sm text-center'>
+                  You need to Update your database. Stop Serpbear and run this command to update your database:
+                  <code className=' bg-gray-700 px-2 py-0 ml-1'>npm run db:migrate</code>
+               </div>
+         )}
          <Head>
             <title>Domains - SerpBear</title>
          </Head>
@@ -71,9 +94,12 @@ const SingleDomain: NextPage = () => {
 
          <div className="flex flex-col w-full max-w-5xl mx-auto p-6 lg:mt-24 lg:p-0">
             <div className='flex justify-between mb-2 items-center'>
-               <div className=' text-sm'>{domainsData?.domains?.length || 0} Domains</div>
+               <div className=' text-sm text-gray-600'>
+                  {domainsData?.domains?.length || 0} Domains <span className=' text-gray-300 ml-1 mr-1'>|</span> {totalKeywords} keywords
+               </div>
                <div>
                   <button
+                  data-testid="addDomainButton"
                   className={'ml-2 inline-block py-2 text-blue-700 font-bold text-sm'}
                   onClick={() => setShowAddDomain(true)}>
                      <span
@@ -88,8 +114,9 @@ const SingleDomain: NextPage = () => {
                            key={domain.ID}
                            domain={domain}
                            selected={false}
-                           isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
+                           isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) || !!domainSCAPiObj[domain.ID] }
                            thumb={domainThumbs[domain.domain]}
+                           updateThumb={manuallyUpdateThumb}
                            // isConsoleIntegrated={false}
                            />;
                })}
@@ -107,16 +134,17 @@ const SingleDomain: NextPage = () => {
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
          <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
              <Settings closeSettings={() => setShowSettings(false)} />
          </CSSTransition>
          <footer className='text-center flex flex-1 justify-center pb-5 items-end'>
-            <span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings?.settings?.version || '0.0.0'}</a></span>
+            <span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings.version || '0.0.0'}</a></span>
          </footer>
+         <Toaster position='bottom-center' containerClassName="react_toaster" />
       </div>
    );
 };
 
-export default SingleDomain;
+export default Domains;

pages/index.tsx

@@ -8,7 +8,7 @@ import Icon from '../components/common/Icon';
 const Home: NextPage = () => {
    const router = useRouter();
    useEffect(() => {
-      router.push('/domains');
+      if (router) router.push('/domains');
    }, [router]);
 
   return (

scrapers/index.ts

@@ -4,6 +4,9 @@ import serpapi from './services/serpapi';
 import serply from './services/serply';
 import spaceserp from './services/spaceserp';
 import proxy from './services/proxy';
+import searchapi from './services/searchapi';
+import valueSerp from './services/valueserp';
+import serper from './services/serper';
 
 export default [
    scrapingRobot,
@@ -12,4 +15,7 @@ export default [
    serply,
    spaceserp,
    proxy,
+   searchapi,
+   valueSerp,
+   serper,
 ];

scrapers/services/searchapi.ts

@@ -0,0 +1,44 @@
+import countries from '../../utils/countries';
+
+interface SearchApiResult {
+   title: string,
+   link: string,
+   position: number,
+ }
+
+const searchapi:ScraperSettings = {
+  id: 'searchapi',
+  name: 'SearchApi.io',
+  website: 'searchapi.io',
+  allowsCity: true,
+  headers: (keyword, settings) => {
+     return {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${settings.scaping_api}`,
+     };
+  },
+  scrapeURL: (keyword) => {
+   const country = keyword.country || 'US';
+   const countryName = countries[country][0];
+   const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
+  },
+  resultObjectKey: 'organic_results',
+  serpExtractor: (content) => {
+     const extractedResult = [];
+     const results: SearchApiResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SearchApiResult[];
+
+     for (const { link, title, position } of results) {
+        if (title && link) {
+           extractedResult.push({
+              title,
+              url: link,
+              position,
+           });
+        }
+     }
+     return extractedResult;
+  },
+};
+
+export default searchapi;

scrapers/services/serpapi.ts

@@ -1,3 +1,5 @@
+import countries from '../../utils/countries';
+
 interface SerpApiResult {
    title: string,
    link: string,
@@ -8,6 +10,7 @@ const serpapi:ScraperSettings = {
    id: 'serpapi',
    name: 'SerpApi.com',
    website: 'serpapi.com',
+   allowsCity: true,
    headers: (keyword, settings) => {
       return {
          'Content-Type': 'application/json',
@@ -15,7 +18,9 @@ const serpapi:ScraperSettings = {
       };
    },
    scrapeURL: (keyword, settings) => {
-      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}&api_key=${settings.scaping_api}`;
+      const countryName = countries[keyword.country || 'US'][0];
+      const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

scrapers/services/serper.ts

@@ -0,0 +1,35 @@
+interface SerperResult {
+   title: string,
+   link: string,
+   position: number,
+}
+
+const serper:ScraperSettings = {
+   id: 'serper',
+   name: 'Serper.dev',
+   website: 'serper.dev',
+   allowsCity: true,
+   scrapeURL: (keyword, settings, countryData) => {
+      const country = keyword.country || 'US';
+      const lang = countryData[country][2];
+      return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
+   },
+   resultObjectKey: 'organic',
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: SerperResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerperResult[];
+
+      for (const { link, title, position } of results) {
+         if (title && link) {
+            extractedResult.push({
+               title,
+               url: link,
+               position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default serper;

scrapers/services/spaceserp.ts

@@ -1,3 +1,5 @@
+import countries from '../../utils/countries';
+
 interface SpaceSerpResult {
    title: string,
    link: string,
@@ -9,10 +11,14 @@ const spaceSerp:ScraperSettings = {
    id: 'spaceSerp',
    name: 'Space Serp',
    website: 'spaceserp.com',
+   allowsCity: true,
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
+      const countryName = countries[country][0];
+      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const device = keyword.device === 'mobile' ? '&device=mobile' : '';
       const lang = countryData[country][2];
-      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${keyword.device === 'mobile' ? '&device=mobile' : ''}&resultBlocks=`;
+      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

scrapers/services/valueserp.ts

@@ -0,0 +1,41 @@
+import countries from '../../utils/countries';
+
+interface ValueSerpResult {
+   title: string,
+   link: string,
+   position: number,
+   domain: string,
+}
+
+const valueSerp:ScraperSettings = {
+   id: 'valueserp',
+   name: 'Value Serp',
+   website: 'valueserp.com',
+   allowsCity: true,
+   scrapeURL: (keyword, settings, countryData) => {
+      const country = keyword.country || 'US';
+      const countryName = countries[country][0];
+      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const device = keyword.device === 'mobile' ? '&device=mobile' : '';
+      const lang = countryData[country][2];
+      console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
+      return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
+   },
+   resultObjectKey: 'organic_results',
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: ValueSerpResult[] = (typeof content === 'string') ? JSON.parse(content) : content as ValueSerpResult[];
+      for (const result of results) {
+         if (result.title && result.link) {
+            extractedResult.push({
+               title: result.title,
+               url: result.link,
+               position: result.position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default valueSerp;

services/domains.tsx

@@ -7,7 +7,7 @@ type UpdatePayload = {
    domain: DomainType
 }
 
-export async function fetchDomains(router: NextRouter, withStats:boolean) {
+export async function fetchDomains(router: NextRouter, withStats:boolean): Promise<{domains: DomainType[]}> {
    const res = await fetch(`${window.location.origin}/api/domains${withStats ? '?withstats=true' : ''}`, { method: 'GET' });
    if (res.status >= 400 && res.status < 600) {
       if (res.status === 401) {
@@ -19,16 +19,67 @@ export async function fetchDomains(router: NextRouter, withStats:boolean) {
    return res.json();
 }
 
+export async function fetchDomain(router: NextRouter, domainName: string): Promise<{domain: DomainType}> {
+   if (!domainName) { throw new Error('No Domain Name Provided!'); }
+   const res = await fetch(`${window.location.origin}/api/domain?domain=${domainName}`, { method: 'GET' });
+   if (res.status >= 400 && res.status < 600) {
+      if (res.status === 401) {
+         console.log('Unauthorized!!');
+         router.push('/login');
+      }
+      throw new Error('Bad response from server');
+   }
+   return res.json();
+}
+
+export async function fetchDomainScreenshot(domain: string, screenshot_key:string, forceFetch = false): Promise<string | false> {
+   const domainThumbsRaw = localStorage.getItem('domainThumbs');
+   const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
+   if (!domThumbs[domain] || forceFetch) {
+      try {
+         const screenshotURL = `https://image.thum.io/get/auth/${screenshot_key}/maxAge/96/width/200/https://${domain}`;
+         const domainImageRes = await fetch(screenshotURL);
+         const domainImageBlob = domainImageRes.status === 200 ? await domainImageRes.blob() : false;
+         if (domainImageBlob) {
+            const reader = new FileReader();
+            await new Promise((resolve, reject) => {
+               reader.onload = resolve;
+               reader.onerror = reject;
+               reader.readAsDataURL(domainImageBlob);
+            });
+            const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
+            localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain]: imageBase }));
+            return imageBase;
+         }
+         return false;
+      } catch (error) {
+         return false;
+      }
+   } else if (domThumbs[domain]) {
+         return domThumbs[domain];
+   }
+
+   return false;
+}
+
 export function useFetchDomains(router: NextRouter, withStats:boolean = false) {
    return useQuery('domains', () => fetchDomains(router, withStats));
 }
 
+export function useFetchDomain(router: NextRouter, domainName:string, onSuccess: Function) {
+   return useQuery('domain', () => fetchDomain(router, domainName), {
+      onSuccess: async (data) => {
+         console.log('Domain Loaded!!!', data.domain);
+         onSuccess(data.domain);
+      } });
+}
+
 export function useAddDomain(onSuccess:Function) {
    const router = useRouter();
    const queryClient = useQueryClient();
-   return useMutation(async (domainName:string) => {
+   return useMutation(async (domains:string[]) => {
       const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' });
-      const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ domain: domainName }) };
+      const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ domains }) };
       const res = await fetch(`${window.location.origin}/api/domains`, fetchOpts);
       if (res.status >= 400 && res.status < 600) {
          throw new Error('Bad response from server');
@@ -37,11 +88,12 @@ export function useAddDomain(onSuccess:Function) {
    }, {
       onSuccess: async (data) => {
          console.log('Domain Added!!!', data);
-         const newDomain:DomainType = data.domain;
-         toast(`${newDomain.domain} Added Successfully!`, { icon: '✔️' });
+         const newDomain:DomainType[] = data.domains;
+         const singleDomain = newDomain.length === 1;
+         toast(`${singleDomain ? newDomain[0].domain : `${newDomain.length} domains`} Added Successfully!`, { icon: '✔️' });
          onSuccess(false);
-         if (newDomain && newDomain.slug) {
-            router.push(`/domain/${data.domain.slug}`);
+         if (singleDomain) {
+            router.push(`/domain/${newDomain[0].slug}`);
          }
          queryClient.invalidateQueries(['domains']);
       },
@@ -58,10 +110,11 @@ export function useUpdateDomain(onSuccess:Function) {
       const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' });
       const fetchOpts = { method: 'PUT', headers, body: JSON.stringify(domainSettings) };
       const res = await fetch(`${window.location.origin}/api/domains?domain=${domain.domain}`, fetchOpts);
+      const responseObj = await res.json();
       if (res.status >= 400 && res.status < 600) {
-         throw new Error('Bad response from server');
+         throw new Error(responseObj?.error || 'Bad response from server');
       }
-      return res.json();
+      return responseObj;
    }, {
       onSuccess: async () => {
          console.log('Settings Updated!!!');
@@ -69,8 +122,8 @@ export function useUpdateDomain(onSuccess:Function) {
          onSuccess();
          queryClient.invalidateQueries(['domains']);
       },
-      onError: () => {
-         console.log('Error Updating Domain Settings!!!');
+      onError: (error) => {
+         console.log('Error Updating Domain Settings!!!', error);
          toast('Error Updating Domain Settings', { icon: '⚠️' });
       },
    });

services/keywords.tsx

@@ -2,16 +2,21 @@ import toast from 'react-hot-toast';
 import { NextRouter } from 'next/router';
 import { useMutation, useQuery, useQueryClient } from 'react-query';
 
-export const fetchKeywords = async (router: NextRouter) => {
-   if (!router.query.slug) { return []; }
-   const res = await fetch(`${window.location.origin}/api/keywords?domain=${router.query.slug}`, { method: 'GET' });
+export const fetchKeywords = async (router: NextRouter, domain: string) => {
+   if (!domain) { return []; }
+   const res = await fetch(`${window.location.origin}/api/keywords?domain=${domain}`, { method: 'GET' });
    return res.json();
 };
 
-export function useFetchKeywords(router: NextRouter, setKeywordSPollInterval?:Function, keywordSPollInterval:undefined|number = undefined) {
+export function useFetchKeywords(
+   router: NextRouter,
+   domain: string,
+   setKeywordSPollInterval?:Function,
+   keywordSPollInterval:undefined|number = undefined,
+) {
    const { data: keywordsData, isLoading: keywordsLoading, isError } = useQuery(
-      ['keywords', router.query.slug],
-      () => fetchKeywords(router),
+      ['keywords', domain],
+      () => fetchKeywords(router, domain),
       {
          refetchInterval: keywordSPollInterval,
          onSuccess: (data) => {
@@ -153,3 +158,23 @@ export function useRefreshKeywords(onSuccess:Function) {
       },
    });
 }
+
+export function useFetchSingleKeyword(keywordID:number) {
+   return useQuery(['keyword', keywordID], async () => {
+      try {
+         const fetchURL = `${window.location.origin}/api/keyword?id=${keywordID}`;
+         const res = await fetch(fetchURL, { method: 'GET' }).then((result) => result.json());
+         if (res.status >= 400 && res.status < 600) {
+            throw new Error('Bad response from server');
+         }
+         return { history: res.keyword.history || [], searchResult: res.keyword.lastResult || [] };
+      } catch (error) {
+         throw new Error('Error Loading Keyword Details');
+      }
+   }, {
+      onError: () => {
+         console.log('Error Loading Keyword Data!!!');
+         toast('Error Loading Keyword Details.', { icon: '⚠️' });
+      },
+   });
+}

services/settings.ts

@@ -10,7 +10,7 @@ export function useFetchSettings() {
    return useQuery('settings', () => fetchSettings());
 }
 
-const useUpdateSettings = (onSuccess:Function|undefined) => {
+export const useUpdateSettings = (onSuccess:Function|undefined) => {
    const queryClient = useQueryClient();
 
    return useMutation(async (settings: SettingsType) => {
@@ -61,4 +61,36 @@ export function useClearFailedQueue(onSuccess:Function) {
    });
 }
 
-export default useUpdateSettings;
+export async function fetchMigrationStatus() {
+   const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'GET' });
+   return res.json();
+}
+
+export function useCheckMigrationStatus() {
+   return useQuery('dbmigrate', () => fetchMigrationStatus());
+}
+
+export const useMigrateDatabase = (onSuccess:Function|undefined) => {
+   const queryClient = useQueryClient();
+
+   return useMutation(async () => {
+      // console.log('settings: ', JSON.stringify(settings));
+      const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'POST' });
+      if (res.status >= 400 && res.status < 600) {
+         throw new Error('Bad response from server');
+      }
+      return res.json();
+   }, {
+      onSuccess: async (res) => {
+         if (onSuccess) {
+            onSuccess(res);
+         }
+         toast('Database Updated!', { icon: '✔️' });
+         queryClient.invalidateQueries(['settings']);
+      },
+      onError: () => {
+         console.log('Error Updating Database!!!');
+         toast('Error Updating Database.', { icon: '⚠️' });
+      },
+   });
+};

styles/globals.css

@@ -282,3 +282,8 @@ body {
       right: 240px;
    }
 }
+
+/* Disable LastPass Icon for Secret Field */
+[autocomplete="off"] + div[data-lastpass-icon-root="true"], [autocomplete="off"] + div[data-lastpass-infield="true"] {
+   display: none;
+}

tsconfig.json

@@ -24,7 +24,8 @@
       "next-env.d.ts",
       "**/*.ts",
       "**/*.tsx",
-      "types.d.ts"
+      "types.d.ts",
+      "./jest.setup.js"
    ],
    "exclude": [
       "node_modules"

types.d.ts

@@ -15,6 +15,7 @@ type DomainType = {
    scVisits?: number,
    scImpressions?: number,
    scPosition?: number,
+   search_console?: string,
 }
 
 type KeywordHistory = {
@@ -39,6 +40,7 @@ type KeywordType = {
    lastUpdateError: {date: string, error: string, scraper: string} | false,
    scData?: KeywordSCData,
    uid?: string
+   city?: string
 }
 
 type KeywordLastResult = {
@@ -61,9 +63,17 @@ type countryCodeData = {
    [ISO:string] : string
 }
 
+type DomainSearchConsole = {
+   property_type: 'domain' | 'url',
+   url: string,
+   client_email:string,
+   private_key:string,
+}
+
 type DomainSettings = {
    notification_interval: string,
    notification_emails: string,
+   search_console?: DomainSearchConsole
 }
 
 type SettingsType = {
@@ -77,13 +87,17 @@ type SettingsType = {
    smtp_port: string,
    smtp_username?: string,
    smtp_password?: string,
-   search_console_integrated?: boolean,
-   available_scapers?: Array,
+   available_scapers?: { label: string, value: string, allowsCity?: boolean }[],
    scrape_interval?: string,
    scrape_delay?: string,
    scrape_retry?: boolean,
    failed_queue?: string[]
-   version?: string
+   version?: string,
+   screenshot_key?: string,
+   search_console: boolean,
+   search_console_client_email: string,
+   search_console_private_key: string,
+   search_console_integrated?: boolean,
 }
 
 type KeywordSCDataChild = {
@@ -107,7 +121,8 @@ type KeywordAddPayload = {
    device: string,
    country: string,
    domain: string,
-   tags: string,
+   tags?: string,
+   city?:string
 }
 
 type SearchAnalyticsRawItem = {
@@ -176,11 +191,23 @@ type scraperExtractedItem = {
    position: number,
 }
 interface ScraperSettings {
+   /** A Unique ID for the Scraper. eg: myScraper */
    id:string,
+   /** The Name of the Scraper */
    name:string,
+   /** The Website address of the Scraper */
    website:string,
+   /** The result object's key that contains the results of the scraped data. For example,
+    * if your scraper API the data like this `{scraped:[item1,item2..]}` the resultObjectKey should be "scraped" */
    resultObjectKey: string,
+   /** If the Scraper allows setting a perices location or allows city level scraping set this to true. */
+   allowsCity?: boolean,
+   /** Set your own custom HTTP header properties when making the scraper API request.
+    * The function should return an object that contains all the header properties you want to pass to API request's header.
+    * Example: `{'Cache-Control': 'max-age=0', 'Content-Type': 'application/json'}` */
    headers?(keyword:KeywordType, settings: SettingsType): Object,
+   /** Construct the API URL for scraping the data through your Scraper's API */
    scrapeURL?(keyword:KeywordType, settings:SettingsType, countries:countryData): string,
+   /** Custom function to extract the serp result from the scraped data. The extracted data should be @return {scraperExtractedItem[]} */
    serpExtractor?(content:string): scraperExtractedItem[],
 }

utils/client/exportcsv.ts

@@ -1,4 +1,4 @@
-import countries from './countries';
+import countries from '../countries';
 
   /**
    * Generates CSV File form the given domain & keywords, and automatically downloads it.

utils/client/helpers.ts

@@ -0,0 +1,2 @@
+/* eslint-disable import/prefer-default-export */
+export const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num);

utils/client/validators.ts

@@ -33,3 +33,14 @@ export const isValidDomain = (domain:string): boolean => {
 
    return isValid;
  };
+
+export const isValidUrl = (str: string) => {
+   let url;
+
+   try {
+     url = new URL(str);
+   } catch (e) {
+     return false;
+   }
+   return url.protocol === 'http:' || url.protocol === 'https:';
+ };

utils/domains.ts

@@ -2,6 +2,12 @@ import Keyword from '../database/models/keyword';
 import parseKeywords from './parseKeywords';
 import { readLocalSCData } from './searchConsole';
 
+/**
+ * The function `getdomainStats` takes an array of domain objects, retrieves keyword and stats data for
+ * each domain, and calculates various statistics for each domain.
+ * @param {DomainType[]} domains - An array of objects of type DomainType.
+ * @returns {DomainType[]} - An array of objects of type DomainType.
+ */
 const getdomainStats = async (domains:DomainType[]): Promise<DomainType[]> => {
    const finalDomains: DomainType[] = [];
    console.log('domains: ', domains.length);
@@ -15,7 +21,8 @@ const getdomainStats = async (domains:DomainType[]): Promise<DomainType[]> => {
       domainWithStat.keywordCount = keywords.length;
       const keywordPositions = keywords.reduce((acc, itm) => (acc + itm.position), 0);
       const KeywordsUpdateDates: number[] = keywords.reduce((acc: number[], itm) => [...acc, new Date(itm.lastUpdated).getTime()], [0]);
-      domainWithStat.keywordsUpdated = new Date(Math.max(...KeywordsUpdateDates)).toJSON();
+      const lastKeywordUpdateDate = Math.max(...KeywordsUpdateDates);
+      domainWithStat.keywordsUpdated = new Date(lastKeywordUpdateDate || new Date(domain.lastUpdated).getTime()).toJSON();
       domainWithStat.avgPosition = Math.round(keywordPositions / keywords.length);
 
       // Then Load the SC File and read the stats and calculate the Last 7 days stats

utils/insight.ts

@@ -1,4 +1,10 @@
-export const sortInsightItems = (items:SCInsightItem[], sortBy: string = 'clicks') => {
+/**
+ * The function `sortInsightItems` sorts an array of `SCInsightItem` objects based on a specified property.
+ * @param {SCInsightItem[]} items - An array of SCInsightItem objects.
+ * @param {string} [sortBy=clicks] - The `sortBy` parameter determines the property by which the `items` array should be sorted.
+ * @returns {SCInsightItem[]}
+ */
+export const sortInsightItems = (items:SCInsightItem[], sortBy: string = 'clicks'): SCInsightItem[] => {
    const sortKey = sortBy as keyof SCInsightItem;
    let sortedItems = [];
    switch (sortKey) {
@@ -18,6 +24,13 @@ export const sortInsightItems = (items:SCInsightItem[], sortBy: string = 'clicks
    return sortedItems;
 };
 
+/**
+ * The `getCountryInsight` function takes search analytics data and returns insights about countries based on clicks, impressions, CTR, and position.
+ * @param {SCDomainDataType} SCData - The SCData parameter is an object that contains search analytics data for different dates.
+ * @param {string} [sortBy=clicks] - The "sortBy" parameter is used to specify the sorting criteria for the country insights.
+ * @param {string} [queryDate=thirtyDays] - The `queryDate` parameter is a string that represents the date range for which the search analytics data is retrieved.
+ * @returns {SCInsightItem[]}
+ */
 export const getCountryInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => {
    const keywordsCounts: { [key:string]: string[] } = {};
    const countryItems: { [key:string]: SCInsightItem } = {};
@@ -57,6 +70,13 @@ export const getCountryInsight = (SCData:SCDomainDataType, sortBy:string = 'clic
    return sortBy ? sortInsightItems(countryInsight, sortBy) : countryInsight;
 };
 
+/**
+ * The `getKeywordsInsight` function takes search analytics data, sorts it based on specified criteria, and returns insights on keywords.
+ * @param {SCDomainDataType} SCData - The SCData parameter is of type SCDomainDataType, which is an object containing search analytics data for a specific domain.
+ * @param {string} [sortBy=clicks] - The "sortBy" parameter is used to specify the sorting criteria for the keyword insights.
+ * @param {string} [queryDate=thirtyDays] - The `queryDate` parameter is a string that represents the date range for which the search analytics data is retrieved.
+ * @returns {SCInsightItem[]}
+ */
 export const getKeywordsInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => {
    const keywordItems: { [key:string]: SCInsightItem } = {};
    const keywordCounts: { [key:string]: number } = {};
@@ -99,6 +119,13 @@ export const getKeywordsInsight = (SCData:SCDomainDataType, sortBy:string = 'cli
    return sortBy ? sortInsightItems(keywordInsight, sortBy) : keywordInsight;
 };
 
+/**
+ * The `getPagesInsight` function takes a domain's search analytics data, sorts it based on specified criteria and returns insights about the pages.
+ * @param {SCDomainDataType} SCData - SCData is an object that contains search analytics data for a  specific domain.
+ * @param {string} [sortBy=clicks] - The `sortBy` parameter is used to specify the sorting criteria for the pages insight.
+ * @param {string} [queryDate=thirtyDays] - The `queryDate` parameter is a string that represents the date range for which you want to retrieve the data.
+ * @returns {SCInsightItem[]}
+ */
 export const getPagesInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => {
    const pagesItems: { [key:string]: SCInsightItem } = {};
    const keywordCounts: { [key:string]: number } = {};

utils/refresh.ts

@@ -17,11 +17,11 @@ const refreshAndUpdateKeywords = async (rawkeyword:Keyword[], settings:SettingsT
    const start = performance.now();
    const updatedKeywords: KeywordType[] = [];
 
-   if (['scrapingant', 'serpapi'].includes(settings.scraper_type)) {
+   if (['scrapingant', 'serpapi', 'searchapi'].includes(settings.scraper_type)) {
       const refreshedResults = await refreshParallel(keywords, settings);
       if (refreshedResults.length > 0) {
          for (const keyword of rawkeyword) {
-            const refreshedkeywordData = refreshedResults.find((k) => k && k.ID === keyword.id);
+            const refreshedkeywordData = refreshedResults.find((k) => k && k.ID === keyword.ID);
             if (refreshedkeywordData) {
                const updatedkeyword = await updateKeywordPosition(keyword, refreshedkeywordData, settings);
                updatedKeywords.push(updatedkeyword);

utils/scraper.ts

@@ -187,15 +187,21 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
 
 /**
  * Find in the domain's position from the extracted search result.
- * @param {string} domain - Domain Name to look for.
+ * @param {string} domainURL - URL Name to look for.
  * @param {SearchResult[]} result - The search result array extracted from the Google Search result.
  * @returns {SERPObject}
  */
-export const getSerp = (domain:string, result:SearchResult[]) : SERPObject => {
-   if (result.length === 0 || !domain) { return { postion: 0, url: '' }; }
+export const getSerp = (domainURL:string, result:SearchResult[]) : SERPObject => {
+   if (result.length === 0 || !domainURL) { return { postion: 0, url: '' }; }
+   const URLToFind = new URL(domainURL.includes('https://') ? domainURL : `https://${domainURL}`);
+   const theURL = URLToFind.hostname + URLToFind.pathname;
+   const isURL = URLToFind.pathname !== '/';
    const foundItem = result.find((item) => {
-      const itemDomain = item.url.replace('www.', '').match(/^(?:https?:)?(?:\/\/)?([^/?]+)/i);
-      return itemDomain && itemDomain.includes(domain.replace('www.', ''));
+      const itemURL = new URL(item.url.includes('https://') ? item.url : `https://${item.url}`);
+      if (isURL && `${theURL}/` === itemURL.hostname + itemURL.pathname) {
+         return true;
+      }
+      return URLToFind.hostname === itemURL.hostname;
    });
    return { postion: foundItem ? foundItem.position : 0, url: foundItem && foundItem.url ? foundItem.url : '' };
 };

utils/searchConsole.ts

@@ -1,19 +1,39 @@
 import { auth, searchconsole_v1 } from '@googleapis/searchconsole';
-import { readFile, writeFile } from 'fs/promises';
+import Cryptr from 'cryptr';
+import { readFile, writeFile, unlink } from 'fs/promises';
 import { getCountryCodeFromAlphaThree } from './countries';
 
 export type SCDomainFetchError = {
    error: boolean,
    errorMsg: string,
 }
+
+type SCAPISettings = { client_email: string, private_key: string }
+
 type fetchConsoleDataResponse = SearchAnalyticsItem[] | SearchAnalyticsStat[] | SCDomainFetchError;
-const fetchSearchConsoleData = async (domainName:string, days:number, type?:string): Promise<fetchConsoleDataResponse> => {
-   if (!domainName) return { error: true, errorMsg: 'Domain Not Provided!' };
+
+/**
+ * Retrieves data from the Google Search Console API based on the provided domain name, number of days, and optional type.
+ * @param {DomainType} domain - The domain for which you want to fetch search console data.
+ * @param {number} days - number of days of data you want to fetch from the Search Console.
+ * @param {string} [type] - (optional) specifies the type of data to fetch from the Search Console.
+ * @param {SCAPISettings} [api] - (optional) specifies the Seach Console API Information.
+ * @returns {Promise<fetchConsoleDataResponse>}
+ */
+const fetchSearchConsoleData = async (domain:DomainType, days:number, type?:string, api?:SCAPISettings): Promise<fetchConsoleDataResponse> => {
+   if (!domain) return { error: true, errorMsg: 'Domain Not Provided!' };
+   if (!api?.private_key || !api?.client_email) return { error: true, errorMsg: 'Search Console API Data Not Avaialable.' };
+   const domainName = domain.domain;
+   const defaultSCSettings = { property_type: 'domain', url: '', client_email: '', private_key: '' };
+   const domainSettings = domain.search_console ? JSON.parse(domain.search_console) : defaultSCSettings;
+   const sCPrivateKey = api?.private_key || process.env.SEARCH_CONSOLE_PRIVATE_KEY || '';
+   const sCClientEmail = api?.client_email || process.env.SEARCH_CONSOLE_CLIENT_EMAIL || '';
+
    try {
    const authClient = new auth.GoogleAuth({
       credentials: {
-        private_key: process.env.SEARCH_CONSOLE_PRIVATE_KEY ? process.env.SEARCH_CONSOLE_PRIVATE_KEY.replaceAll('\\n', '\n') : '',
-        client_email: process.env.SEARCH_CONSOLE_CLIENT_EMAIL ? process.env.SEARCH_CONSOLE_CLIENT_EMAIL : '',
+        private_key: (sCPrivateKey).replaceAll('\\n', '\n'),
+        client_email: (sCClientEmail || '').trim(),
       },
       scopes: [
         'https://www.googleapis.com/auth/webmasters.readonly',
@@ -42,7 +62,8 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
       };
    }
 
-      const res = client.searchanalytics.query({ siteUrl: `sc-domain:${domainName}`, requestBody });
+      const siteUrl = domainSettings.property_type === 'url' && domainSettings.url ? domainSettings.url : `sc-domain:${domainName}`;
+      const res = client.searchanalytics.query({ siteUrl, requestBody });
       const resData:any = (await res).data;
       let finalRows = resData.rows ? resData.rows.map((item:SearchAnalyticsRawItem) => parseSearchConsoleItem(item, domainName)) : [];
 
@@ -61,19 +82,29 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
       }
 
       return finalRows;
-   } catch (error:any) {
+   } catch (err:any) {
       const qType = type === 'stats' ? '(stats)' : `(${days}days)`;
-      console.log(`[ERROR] Search Console API Error for ${domainName} ${qType} : `, error?.response?.status, error?.response?.statusText);
-      return { error: true, errorMsg: `${error?.response?.status}: ${error?.response?.statusText}` };
+      const errorMsg = err?.response?.status && `${err?.response?.statusText}. ${err?.response?.data?.error_description}`;
+      console.log(`[ERROR] Search Console API Error for ${domainName} ${qType} : `, errorMsg || err?.code);
+      // console.log('SC ERROR :', err);
+      return { error: true, errorMsg: errorMsg || err?.code };
    }
 };
 
-export const fetchDomainSCData = async (domain:string): Promise<SCDomainDataType> => {
+/**
+ * The function fetches search console data for a given domain and returns it in a structured format.
+ * @param {DomainType} domain - The `domain` parameter is a Domain object that represents the domain for which we
+ * want to fetch search console data.
+ * @returns The function `fetchDomainSCData` is returning a Promise that resolves to an object of type
+ * `SCDomainDataType`.
+ */
+export const fetchDomainSCData = async (domain:DomainType, scAPI?: SCAPISettings): Promise<SCDomainDataType> => {
    const days = [3, 7, 30];
    const scDomainData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '', stats: [] };
-   if (domain) {
+   if (domain.domain && scAPI) {
+      const theDomain = domain;
       for (const day of days) {
-         const items = await fetchSearchConsoleData(domain, day);
+         const items = await fetchSearchConsoleData(theDomain, day, undefined, scAPI);
           scDomainData.lastFetched = new Date().toJSON();
          if (Array.isArray(items)) {
             if (day === 3) scDomainData.threeDays = items as SearchAnalyticsItem[];
@@ -83,16 +114,22 @@ export const fetchDomainSCData = async (domain:string): Promise<SCDomainDataType
             scDomainData.lastFetchError = items.errorMsg;
          }
       }
-      const stats = await fetchSearchConsoleData(domain, 30, 'stat');
+      const stats = await fetchSearchConsoleData(theDomain, 30, 'stat', scAPI);
       if (stats && Array.isArray(stats) && stats.length > 0) {
          scDomainData.stats = stats as SearchAnalyticsStat[];
       }
-      await updateLocalSCData(domain, scDomainData);
+      await updateLocalSCData(domain.domain, scDomainData);
    }
 
    return scDomainData;
 };
 
+/**
+ * The function takes a raw search console item and a domain name as input and returns a parsed search analytics item.
+ * @param {SearchAnalyticsRawItem} SCItem - The SCItem parameter is an object that represents a raw item from the Search Console API.
+ * @param {string} domainName - The `domainName` parameter is a string that represents the domain name of the website.
+ * @returns {SearchAnalyticsItem}.
+ */
 export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainName: string): SearchAnalyticsItem => {
    const { clicks = 0, impressions = 0, ctr = 0, position = 0 } = SCItem;
    const keyword = SCItem.keys[0];
@@ -104,6 +141,12 @@ export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainNam
    return { keyword, uid, device, country, clicks, impressions, ctr: ctr * 100, position, page };
 };
 
+/**
+ * The function integrates search console data with a keyword object and returns the updated keyword object with the search console data.
+ * @param {KeywordType} keyword - The `keyword` parameter is of type `KeywordType`, which is a custom type representing a keyword.
+ * @param {SCDomainDataType} SCData - SCData is an object that contains search analytics data for different time periods
+ * @returns {KeywordType}
+ */
 export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainDataType) : KeywordType => {
    const kuid = `${keyword.country.toLowerCase()}:${keyword.device}:${keyword.keyword.replaceAll(' ', '_')}`;
    const impressions:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 };
@@ -111,9 +154,9 @@ export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainData
    const ctr:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 };
    const position:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 };
 
-   const threeDaysData = SCData.threeDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
-   const SevenDaysData = SCData.sevenDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
-   const ThirdyDaysData = SCData.thirtyDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
+   const threeDaysData = SCData?.threeDays?.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
+   const SevenDaysData = SCData?.sevenDays?.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
+   const ThirdyDaysData = SCData?.thirtyDays?.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
    const totalData:any = { threeDays: threeDaysData, sevenDays: SevenDaysData, thirtyDays: ThirdyDaysData };
 
    Object.keys(totalData).forEach((dataKey) => {
@@ -136,18 +179,103 @@ export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainData
    return { ...keyword, scData: finalSCData };
 };
 
-export const readLocalSCData = async (domain:string): Promise<SCDomainDataType> => {
-   const filePath = `${process.cwd()}/data/SC_${domain}.json`;
-   const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch(async () => { await updateLocalSCData(domain); return '{}'; });
-   const domainSCData = JSON.parse(currentQueueRaw);
-   return domainSCData;
+/**
+ * Retrieves the Search Console API information for a given domain.
+ * @param {DomainType} domain - The `domain` parameter is of type `DomainType`, which represents a
+ * domain object. It likely contains information about a specific domain, such as its name, search
+ * console settings, etc.
+ * @returns an object of type `SCAPISettings`.
+ */
+export const getSearchConsoleApiInfo = async (domain: DomainType): Promise<SCAPISettings> => {
+   const scAPIData = { client_email: '', private_key: '' };
+   // Check if the Domain Has the API Data
+   const domainSCSettings = domain.search_console && JSON.parse(domain.search_console);
+   if (domainSCSettings && domainSCSettings.private_key) {
+      if (!domainSCSettings.private_key.includes('BEGIN PRIVATE KEY')) {
+         const cryptr = new Cryptr(process.env.SECRET as string);
+         scAPIData.client_email = domainSCSettings.client_email ? cryptr.decrypt(domainSCSettings.client_email) : '';
+         scAPIData.private_key = domainSCSettings.private_key ? cryptr.decrypt(domainSCSettings.private_key) : '';
+      } else {
+         scAPIData.client_email = domainSCSettings.client_email;
+         scAPIData.private_key = domainSCSettings.private_key;
+      }
+   }
+   // Check if the App Settings Has the API Data
+   if (!scAPIData?.private_key) {
+      const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
+      const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {};
+      const cryptr = new Cryptr(process.env.SECRET as string);
+      scAPIData.client_email = settings.search_console_client_email ? cryptr.decrypt(settings.search_console_client_email) : '';
+      scAPIData.private_key = settings.search_console_private_key ? cryptr.decrypt(settings.search_console_private_key) : '';
+   }
+   if (!scAPIData?.private_key && process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
+      scAPIData.client_email = process.env.SEARCH_CONSOLE_CLIENT_EMAIL;
+      scAPIData.private_key = process.env.SEARCH_CONSOLE_PRIVATE_KEY;
+   }
+
+   return scAPIData;
 };
 
+/**
+ * Checks if the provided domain level Google Search Console API info is valid.
+ * @param {DomainType} domain - The domain that represents the domain for which the SC API info is being checked.
+ * @returns an object of type `{ isValid: boolean, error: string }`.
+ */
+export const checkSerchConsoleIntegration = async (domain: DomainType): Promise<{ isValid: boolean, error: string }> => {
+   const res = { isValid: false, error: '' };
+   const { client_email = '', private_key = '' } = domain?.search_console ? JSON.parse(domain.search_console) : {};
+   const response = await fetchSearchConsoleData(domain, 3, undefined, { client_email, private_key });
+   if (Array.isArray(response)) { res.isValid = true; }
+   if ((response as SCDomainFetchError)?.errorMsg) { res.error = (response as SCDomainFetchError).errorMsg; }
+   return res;
+};
+
+/**
+ * The function reads and returns the domain-specific data stored in a local JSON file.
+ * @param {string} domain - The `domain` parameter is a string that represents the domain for which the SC data is being read.
+ * @returns {Promise<SCDomainDataType>}
+ */
+export const readLocalSCData = async (domain:string): Promise<SCDomainDataType|false> => {
+   try {
+      const filePath = `${process.cwd()}/data/SC_${domain.replaceAll('/', '-')}.json`;
+      const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch(async () => { await updateLocalSCData(domain); return '{}'; });
+      const domainSCData = JSON.parse(currentQueueRaw);
+      return domainSCData;
+   } catch (error) {
+      return false;
+   }
+};
+
+/**
+ * The function reads and returns the domain-specific data stored in a local JSON file.
+ * @param {string} domain - The `domain` parameter is a string that represents the domain for which the SC data will be written.
+ * @param {SCDomainDataType} scDomainData - an object that contains search analytics data for different time periods.
+ * @returns {Promise<SCDomainDataType|false>}
+ */
 export const updateLocalSCData = async (domain:string, scDomainData?:SCDomainDataType): Promise<SCDomainDataType|false> => {
-   const filePath = `${process.cwd()}/data/SC_${domain}.json`;
-   const emptyData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '' };
-   await writeFile(filePath, JSON.stringify(scDomainData || emptyData), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
-   return scDomainData || emptyData;
+   try {
+      const filePath = `${process.cwd()}/data/SC_${domain.replaceAll('/', '-')}.json`;
+      const emptyData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '' };
+      await writeFile(filePath, JSON.stringify(scDomainData || emptyData), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
+      return scDomainData || emptyData;
+   } catch (error) {
+      return false;
+   }
+};
+
+/**
+ * The function removes the domain-specific Seach Console data stored in a local JSON file.
+ * @param {string} domain - The `domain` parameter is a string that represents the domain for which the SC data file will be removed.
+ * @returns {Promise<boolean>} - Returns true if file was removed, else returns false.
+ */
+export const removeLocalSCData = async (domain:string): Promise<boolean> => {
+   const filePath = `${process.cwd()}/data/SC_${domain.replaceAll('/', '-')}.json`;
+   try {
+      await unlink(filePath);
+      return true;
+   } catch (error) {
+      return false;
+   }
 };
 
 export default fetchSearchConsoleData;