chore(release): 1.0.2

closes #164

.eslintrc.json

@@ -12,6 +12,8 @@
    "no-await-in-loop": "off",
    "arrow-body-style":"off",
    "max-len": ["error", {"code": 150, "ignoreComments": true, "ignoreUrls": true}],
+   "import/no-extraneous-dependencies": "off",
+   "no-unused-vars": "off",
    "import/extensions": [
       "error",
       "ignorePackages",

.sequelizerc

@@ -0,0 +1,8 @@
+const path = require('path');
+
+module.exports = {
+  'config': path.resolve('database', 'config.js'),
+  'models-path': path.resolve('database', 'models'),
+  'seeders-path': path.resolve('database', 'seeders'),
+  'migrations-path': path.resolve('database', 'migrations')
+};

CHANGELOG.md

@@ -2,6 +2,67 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [1.0.2](https://github.com/towfiqi/serpbear/compare/v1.0.1...v1.0.2) (2024-02-15)
+
+
+### Bug Fixes
+
+* Resolves Incorrect Position issue. ([0e64b95](https://github.com/towfiqi/serpbear/commit/0e64b95cd5303525535ea84a77181281d7f5618e)), closes [#164](https://github.com/towfiqi/serpbear/issues/164)
+
+### [1.0.1](https://github.com/towfiqi/serpbear/compare/v1.0.0...v1.0.1) (2024-02-13)
+
+
+### Bug Fixes
+
+* Resolves the app crash issue when there is no database. ([e5dd411](https://github.com/towfiqi/serpbear/commit/e5dd411aa9aef58ebb226f2b793a2632ab9069a7)), closes [#161](https://github.com/towfiqi/serpbear/issues/161) [#162](https://github.com/towfiqi/serpbear/issues/162)
+
+## [1.0.0](https://github.com/towfiqi/serpbear/compare/v0.3.4...v1.0.0) (2024-02-09)
+
+
+### Features
+
+* Adds Serper.dev integration ([b4ad69b](https://github.com/towfiqi/serpbear/commit/b4ad69baaa0f865938f8b0eace6732a9e6b1b381)), closes [#138](https://github.com/towfiqi/serpbear/issues/138)
+* Adds the ability for city level scraping for scapers that allow it. ([3719f21](https://github.com/towfiqi/serpbear/commit/3719f21d98d173219cef5656579fa0e5340ccdbf)), closes [#139](https://github.com/towfiqi/serpbear/issues/139) [#151](https://github.com/towfiqi/serpbear/issues/151)
+* adds the ability to add url as a domain. ([3c2a1b8](https://github.com/towfiqi/serpbear/commit/3c2a1b8a5b8a2a4a2179a5031582f8202c2e494a)), closes [#53](https://github.com/towfiqi/serpbear/issues/53) [#90](https://github.com/towfiqi/serpbear/issues/90) [#119](https://github.com/towfiqi/serpbear/issues/119)
+* Adds the Ability to set Search Console Property type via Domain Settings. ([b2e97b2](https://github.com/towfiqi/serpbear/commit/b2e97b2ebec380f0edf7ddc0640c2126eff006ac)), closes [#50](https://github.com/towfiqi/serpbear/issues/50)
+* Adds the ability to setup Search Console through the UI. ([f04b10c](https://github.com/towfiqi/serpbear/commit/f04b10cf6b065e3023965112a60e0aa702212a4b)), closes [#59](https://github.com/towfiqi/serpbear/issues/59) [#146](https://github.com/towfiqi/serpbear/issues/146)
+* Adds ValueSerp Integration. ([1041cb3](https://github.com/towfiqi/serpbear/commit/1041cb3c0bb69e9034696624e03433be28e83ac6)), closes [#105](https://github.com/towfiqi/serpbear/issues/105) [#106](https://github.com/towfiqi/serpbear/issues/106)
+
+
+### Bug Fixes
+
+* Resolves Keywords filter crashing issue. ([633ab2c](https://github.com/towfiqi/serpbear/commit/633ab2c467be5b7b86d4547ae0c59034e595a42d))
+* Resolves missing Keyword Loading Spinner issue. ([dbf540c](https://github.com/towfiqi/serpbear/commit/dbf540cfdb16ddb02c9d26618e3680d34799f57f))
+
+### [0.3.4](https://github.com/towfiqi/serpbear/compare/v0.3.3...v0.3.4) (2024-01-15)
+
+
+### Features
+
+* adds ability to add multiple domains at once. ([faa88c9](https://github.com/towfiqi/serpbear/commit/faa88c92542194f19b5cfe2b5cfd07d7d4f7ee46))
+* Adds the ability to show/hide Keys & Passwords in Settings Panel ([c897a52](https://github.com/towfiqi/serpbear/commit/c897a525509baf5b9e8df18d82f5e87aec64f66e))
+
+
+### Bug Fixes
+
+* fixes local SC data not being removed on deleting domain. ([cca9f95](https://github.com/towfiqi/serpbear/commit/cca9f95358b2d3ea06edb33595cdbf616a175469))
+* Resolves incorrect keyword average SC data values in Tracker ([e166b58](https://github.com/towfiqi/serpbear/commit/e166b588aa6c8db55d61b5bc13db66514575c745))
+* resolves newly added Domain's Update time rendering issue ([df3a738](https://github.com/towfiqi/serpbear/commit/df3a738788fa957e7246a0feefe395a9eadd5baf))
+
+### [0.3.3](https://github.com/towfiqi/serpbear/compare/v0.3.2...v0.3.3) (2023-11-12)
+
+
+### Features
+
+* Adds ability to visit pages from Insight tab ([60c68bd](https://github.com/towfiqi/serpbear/commit/60c68bd339db7aeed35aea035dd21691702ffee3))
+* Domains now show their favicon. ([2339e31](https://github.com/towfiqi/serpbear/commit/2339e31af9e90bf918f5bcd4f23114f38cef0313)), closes [#130](https://github.com/towfiqi/serpbear/issues/130)
+* Shows total keywords count in domains page ([fbd23ed](https://github.com/towfiqi/serpbear/commit/fbd23ede256062c72ec2f7e3983a0a02f0240725))
+
+
+### Bug Fixes
+
+* Resolves Website Thumbnail missing issue ([4a60271](https://github.com/towfiqi/serpbear/commit/4a60271cac1209dc02748c4d31943bb21c9ecaf2)), closes [#131](https://github.com/towfiqi/serpbear/issues/131)
+
 ### [0.3.2](https://github.com/towfiqi/serpbear/compare/v0.3.1...v0.3.2) (2023-11-09)
 
 

Dockerfile

@@ -11,7 +11,8 @@ FROM node:lts-alpine AS builder
 WORKDIR /app
 COPY --from=deps /app ./
 RUN rm -rf /app/data
-RUN rm -rf /app/__test__
+RUN rm -rf /app/__tests__
+RUN rm -rf /app/__mocks__
 RUN npm run build
 
 
@@ -29,13 +30,17 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 # setup the cron
 COPY --from=builder --chown=nextjs:nodejs /app/cron.js ./
 COPY --from=builder --chown=nextjs:nodejs /app/email ./email
+COPY --from=builder --chown=nextjs:nodejs /app/database ./database
+COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc
+COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv croner @googleapis/searchconsole
+RUN npm i cryptr dotenv croner @googleapis/searchconsole sequelize-cli
 RUN npm i -g concurrently
 
 USER nextjs
 
 EXPOSE 3000
 
+ENTRYPOINT ["/app/entrypoint.sh"]
 CMD ["concurrently","node server.js", "node cron.js"]

README.md

@@ -42,6 +42,7 @@ The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, Searc
 | serpapi.com | From $50/mo** | From 5,000/mo** | Yes |
 | spaceserp.com | $59/lifetime | 15,000/mo | Yes |
 | SearchApi.io | From $40/mo | From 10,000/mo | Yes |
+| valueserp.com | Pay As You Go | $2.50/1000 req | No |
 
 (*) Free upto a limit. If you are using ScrapingAnt you can lookup 10,000 times per month for free.
 (**) Free up to 100 per month. Paid from 5,000 to 10,000,000+ per month.

__mocks__/data.ts

@@ -2,10 +2,11 @@ export const dummyDomain = {
    ID: 1,
    domain: 'compressimage.io',
    slug: 'compressimage-io',
-   keywordCount: 0,
+   keywordCount: 10,
+   avgPosition: 24,
    lastUpdated: '2022-11-11T10:00:32.243',
    added: '2022-11-11T10:00:32.244',
-   tags: [],
+   tags: '',
    notification: true,
    notification_interval: 'daily',
    notification_emails: '',
@@ -33,7 +34,7 @@ export const dummyKeywords = [
        lastResult: [],
        sticky: false,
        updating: false,
-       lastUpdateError: 'false',
+       lastUpdateError: false as false,
    },
    {
        ID: 2,
@@ -56,6 +57,23 @@ export const dummyKeywords = [
        lastResult: [],
        sticky: false,
        updating: false,
-       lastUpdateError: 'false',
+       lastUpdateError: false as false,
    },
 ];
+
+export const dummySettings = {
+   scaping_api: '',
+   scraper_type: 'none',
+   notification_interval: 'never',
+   notification_email: '',
+   notification_email_from: '',
+   smtp_server: '',
+   smtp_port: '',
+   smtp_username: '',
+   smtp_password: '',
+   scrape_retry: false,
+   search_console_integrated: false,
+   screenshot_key: '',
+   available_scapers: [],
+   failed_queue: [],
+};

__test__/components/Modal.test.tsx

@@ -1,35 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import React from 'react';
-import Modal from '../../components/common/Modal';
-
-// jest.mock('React', () => ({
-//    ...jest.requireActual('React'),
-//    useEffect: jest.fn(),
-// }));
-// const mockedUseEffect = useEffect as jest.Mock<any>;
-
-// jest.mock('../../components/common/Icon', () => () => <div data-testid="icon"/>);
-
-describe('Modal Component', () => {
-   it('Renders without crashing', async () => {
-       render(<Modal closeModal={() => console.log() }><div></div></Modal>);
-      //  mockedUseEffect.mock.calls[0]();
-       expect(document.querySelector('.modal')).toBeInTheDocument();
-   });
-//    it('Sets up the escapae key shortcut', async () => {
-//       render(<Modal closeModal={() => console.log() }><div></div></Modal>);
-//       expect(mockedUseEffect).toBeCalled();
-//   });
-   it('Displays the Given Content', async () => {
-      render(<Modal closeModal={() => console.log() }>
-        <div>
-           <h1>Hello Modal!!</h1>
-        </div>
-      </Modal>);
-      expect(await screen.findByText('Hello Modal!!')).toBeInTheDocument();
-   });
-   it('Renders Modal Title', async () => {
-      render(<Modal closeModal={() => console.log() } title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
-      expect(await screen.findByText('Sample Modal Title')).toBeInTheDocument();
-   });
-});

__test__/components/Sidebar.test.tsx

@@ -1,11 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import Sidebar from '../../components/common/Sidebar';
-
-describe('Sidebar Component', () => {
-   it('renders without crashing', async () => {
-       render(<Sidebar domains={[]} showAddModal={() => console.log() } />);
-       expect(
-           await screen.findByText('SerpBear'),
-       ).toBeInTheDocument();
-   });
-});

__test__/hooks/domains.tests.tsx

@@ -1,21 +0,0 @@
-import { waitFor } from '@testing-library/react';
-// import { useFetchDomains } from '../../services/domains';
-// import { createWrapper } from '../utils';
-
-jest.mock('next/router', () => ({
-   useRouter: () => ({
-     query: { slug: 'compressimage-io' },
-     push: (link:string) => { console.log('Pushed', link); },
-   }),
-}));
-
-describe('DomainHooks', () => {
-   it('useFetchDomains should fetch the Domains', async () => {
-      // const { result } = renderHook(() => useFetchDomains(), { wrapper: createWrapper() });
-      const result = { current: { isSuccess: false, data: '' } };
-      await waitFor(() => {
-         console.log('result.current: ', result.current.data);
-         return expect(result.current.isSuccess).toBe(true);
-      });
-   });
-});

__tests__/components/DomainItem.test.tsx

@@ -0,0 +1,35 @@
+import { fireEvent, render } from '@testing-library/react';
+import DomainItem from '../../components/domains/DomainItem';
+import { dummyDomain } from '../../__mocks__/data';
+
+const updateThumbMock = jest.fn();
+const domainItemProps = {
+   domain: dummyDomain,
+   selected: false,
+   isConsoleIntegrated: false,
+   thumb: '',
+   updateThumb: updateThumbMock,
+};
+
+describe('DomainItem Component', () => {
+   it('renders without crashing', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      expect(container.querySelector('.domItem')).toBeInTheDocument();
+   });
+   it('renders keywords count', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const domStatskeywords = container.querySelector('.dom_stats div:nth-child(1)');
+      expect(domStatskeywords?.textContent).toBe('Keywords10');
+   });
+   it('renders avg position', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const domStatsAvg = container.querySelector('.dom_stats div:nth-child(2)');
+      expect(domStatsAvg?.textContent).toBe('Avg position24');
+   });
+   it('updates domain thumbnail on relevant button click', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const reloadThumbbBtn = container.querySelector('.domain_thumb button');
+      if (reloadThumbbBtn) fireEvent.click(reloadThumbbBtn);
+      expect(updateThumbMock).toHaveBeenCalledWith(dummyDomain.domain);
+   });
+});

__tests__/components/Keyword.test.tsx

@@ -1,8 +1,14 @@
 import { fireEvent, render, screen } from '@testing-library/react';
 import Keyword from '../../components/keywords/Keyword';
-import { dummyKeywords } from '../data';
+import { dummyKeywords } from '../../__mocks__/data';
 
-const keywordFunctions = {
+const keywordProps = {
+   keywordData: dummyKeywords[0],
+   selected: false,
+   index: 0,
+   showSCData: false,
+   scDataType: '',
+   style: {},
    refreshkeyword: jest.fn(),
    favoriteKeyword: jest.fn(),
    removeKeyword: jest.fn(),
@@ -10,35 +16,37 @@ const keywordFunctions = {
    manageTags: jest.fn(),
    showKeywordDetails: jest.fn(),
 };
-
+jest.mock('react-chartjs-2', () => ({
+   Line: () => null,
+ }));
 describe('Keyword Component', () => {
    it('renders without crashing', async () => {
-       render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+       render(<Keyword {...keywordProps} />);
        expect(await screen.findByText('compress image')).toBeInTheDocument();
    });
    it('Should Render Position Correctly', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_position');
       expect(positionElement?.childNodes[0].nodeValue).toBe('19');
    });
    it('Should Display Position Change arrow', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_position i');
       expect(positionElement?.textContent).toBe('▲ 1');
    });
    it('Should Display the SERP Page URL', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_url');
       expect(positionElement?.textContent).toBe('/');
    });
    it('Should Display the Keyword Options on dots Click', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
-      const button = document.querySelector('.keyword .keyword_dots');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      const { container } = render(<Keyword {...keywordProps} />);
+      const button = container.querySelector('.keyword_dots');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.keyword_options')).toBeVisible();
    });
    // it('Should favorite Keywords', async () => {
-   //    render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+   //    render(<Keyword {...keywordProps} />);
    //    const button = document.querySelector('.keyword .keyword_dots');
    //    if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
    //    const option = document.querySelector('.keyword .keyword_options li:nth-child(1) a');

__tests__/components/Modal.test.tsx

@@ -0,0 +1,28 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import Modal from '../../components/common/Modal';
+
+const closeModalMock = jest.fn();
+describe('Modal Component', () => {
+   it('Renders without crashing', async () => {
+       render(<Modal closeModal={closeModalMock }><div></div></Modal>);
+       expect(document.querySelector('.modal')).toBeInTheDocument();
+   });
+   it('Displays the Given Content', async () => {
+      render(<Modal closeModal={closeModalMock}>
+        <div>
+           <h1>Hello Modal!!</h1>
+        </div>
+      </Modal>);
+      expect(await screen.findByText('Hello Modal!!')).toBeInTheDocument();
+   });
+   it('Renders Modal Title', async () => {
+      render(<Modal closeModal={closeModalMock} title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
+      expect(await screen.findByText('Sample Modal Title')).toBeInTheDocument();
+   });
+   it('Closes the modal on close button click', async () => {
+      const { container } = render(<Modal closeModal={closeModalMock} title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
+      const closeBtn = container.querySelector('.modal-close');
+      if (closeBtn) fireEvent.click(closeBtn);
+      expect(closeModalMock).toHaveBeenCalled();
+   });
+});

__tests__/components/Sidebar.test.tsx

@@ -0,0 +1,23 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import Sidebar from '../../components/common/Sidebar';
+import { dummyDomain } from '../../__mocks__/data';
+
+const addDomainMock = jest.fn();
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+
+describe('Sidebar Component', () => {
+   it('renders without crashing', async () => {
+       render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+       expect(screen.getByText('SerpBear')).toBeInTheDocument();
+   });
+   it('renders domain list', async () => {
+      render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+      expect(screen.getByText('compressimage.io')).toBeInTheDocument();
+   });
+   it('calls showAddModal on Add Domain button click', async () => {
+      render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+      const addDomainBtn = screen.getByTestId('add_domain');
+      fireEvent.click(addDomainBtn);
+      expect(addDomainMock).toHaveBeenCalledWith(true);
+   });
+});

__tests__/components/Topbar.test.tsx

@@ -1,9 +1,15 @@
 import { render, screen } from '@testing-library/react';
 import TopBar from '../../components/common/TopBar';
 
+jest.mock('next/router', () => ({
+   useRouter: () => ({
+      pathname: '/',
+   }),
+}));
+
 describe('TopBar Component', () => {
    it('renders without crashing', async () => {
-       render(<TopBar showSettings={() => console.log() } />);
+       render(<TopBar showSettings={jest.fn} showAddModal={jest.fn} />);
        expect(
            await screen.findByText('SerpBear'),
        ).toBeInTheDocument();

__tests__/hooks/domains.test.tsx

@@ -0,0 +1,27 @@
+import { renderHook, waitFor } from '@testing-library/react';
+import mockRouter from 'next-router-mock';
+
+import { useFetchDomains } from '../../services/domains';
+import { createWrapper } from '../../__mocks__/utils';
+import { dummyDomain } from '../../__mocks__/data';
+
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+
+fetchMock.mockIf(`${window.location.origin}/api/domains`, async () => {
+   return new Promise((resolve) => {
+      resolve({
+         body: JSON.stringify({ domains: [dummyDomain] }),
+         status: 200,
+      });
+   });
+});
+
+describe('DomainHooks', () => {
+   it('useFetchDomains should fetch the Domains', async () => {
+      const { result } = renderHook(() => useFetchDomains(mockRouter), { wrapper: createWrapper() });
+      // const result = { current: { isSuccess: false, data: '' } };
+      await waitFor(() => {
+         return expect(result.current.isLoading).toBe(false);
+      });
+   });
+});

__tests__/pages/domain.test.tsx

@@ -1,17 +1,26 @@
 import { fireEvent, render, screen } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from 'react-query';
 import SingleDomain from '../../pages/domain/[slug]';
 import { useAddDomain, useDeleteDomain, useFetchDomains, useUpdateDomain } from '../../services/domains';
-import { useAddKeywords, useDeleteKeywords, useFavKeywords, useFetchKeywords, useRefreshKeywords } from '../../services/keywords';
-import { dummyDomain, dummyKeywords } from '../data';
+import { useAddKeywords, useDeleteKeywords,
+   useFavKeywords, useFetchKeywords, useRefreshKeywords, useFetchSingleKeyword } from '../../services/keywords';
+import { dummyDomain, dummyKeywords, dummySettings } from '../../__mocks__/data';
+import { useFetchSettings } from '../../services/settings';
 
 jest.mock('../../services/domains');
 jest.mock('../../services/keywords');
+jest.mock('../../services/settings');
+
 jest.mock('next/router', () => ({
    useRouter: () => ({
      query: { slug: dummyDomain.slug },
    }),
 }));
 
+jest.mock('react-chartjs-2', () => ({
+   Line: () => null,
+}));
+
 const useFetchDomainsFunc = useFetchDomains as jest.Mock<any>;
 const useFetchKeywordsFunc = useFetchKeywords as jest.Mock<any>;
 const useDeleteKeywordsFunc = useDeleteKeywords as jest.Mock<any>;
@@ -21,11 +30,17 @@ const useAddDomainFunc = useAddDomain as jest.Mock<any>;
 const useAddKeywordsFunc = useAddKeywords as jest.Mock<any>;
 const useUpdateDomainFunc = useUpdateDomain as jest.Mock<any>;
 const useDeleteDomainFunc = useDeleteDomain as jest.Mock<any>;
+const useFetchSettingsFunc = useFetchSettings as jest.Mock<any>;
+const useFetchSingleKeywordFunc = useFetchSingleKeyword as jest.Mock<any>;
 
 describe('SingleDomain Page', () => {
+   const queryClient = new QueryClient();
    beforeEach(() => {
+      useFetchSettingsFunc.mockImplementation(() => ({ data: { settings: dummySettings }, isLoading: false }));
       useFetchDomainsFunc.mockImplementation(() => ({ data: { domains: [dummyDomain] }, isLoading: false }));
       useFetchKeywordsFunc.mockImplementation(() => ({ keywordsData: { keywords: dummyKeywords }, keywordsLoading: false }));
+      const fetchPayload = { history: dummyKeywords[0].history || [], searchResult: dummyKeywords[0].lastResult || [] };
+      useFetchSingleKeywordFunc.mockImplementation(() => ({ data: fetchPayload, isLoading: false }));
       useDeleteKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
       useFavKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
       useRefreshKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
@@ -38,158 +53,154 @@ describe('SingleDomain Page', () => {
       jest.clearAllMocks();
    });
    it('Render without crashing.', async () => {
-      const { getByTestId } = render(<SingleDomain />);
-      // screen.debug(undefined, Infinity);
-      expect(getByTestId('domain-header')).toBeInTheDocument();
-      // expect(await result.findByText(/compressimage/i)).toBeInTheDocument();
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      expect(screen.getByTestId('domain-header')).toBeInTheDocument();
    });
    it('Should Call the useFetchDomains hook on render.', async () => {
-      render(<SingleDomain />);
-      // screen.debug(undefined, Infinity);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       expect(useFetchDomains).toHaveBeenCalled();
-      // expect(await result.findByText(/compressimage/i)).toBeInTheDocument();
    });
    it('Should Render the Keywords', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(2);
    });
    it('Should Display the Keywords Details Sidebar on Keyword Click.', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const keywords = document.querySelectorAll('.keyword');
       const firstKeyword = keywords && keywords[0].querySelector('a');
-      if (firstKeyword) fireEvent(firstKeyword, new MouseEvent('click', { bubbles: true }));
+      if (firstKeyword) fireEvent.click(firstKeyword);
+      expect(useFetchSingleKeyword).toHaveBeenCalled();
       expect(screen.getByTestId('keywordDetails')).toBeVisible();
    });
    it('Should Display the AddDomain Modal on Add Domain Button Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=add_domain]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('add_domain');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('adddomain_modal')).toBeVisible();
    });
    it('Should Display the AddKeywords Modal on Add Keyword Button Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=add_keyword]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('add_keyword');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('addkeywords_modal')).toBeVisible();
    });
 
    it('Should display the Domain Settings on Settings Button click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=show_domain_settings]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('show_domain_settings');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('domain_settings')).toBeVisible();
    });
 
    it('Device Tab change should be functioning.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=mobile_tab]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('mobile_tab');
+      if (button) fireEvent.click(button);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(0);
    });
 
    it('Search Filter should function properly', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const inputNode = screen.getByTestId('filter_input');
-      fireEvent.change(inputNode, { target: { value: 'compressor' } }); // triggers onChange event
+      if (inputNode) fireEvent.change(inputNode, { target: { value: 'compressor' } }); // triggers onChange event
       expect(inputNode.getAttribute('value')).toBe('compressor');
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(1);
    });
 
    it('Country Filter should function properly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=filter_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('filter_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.country_filter')).toBeVisible();
 
       const countrySelect = document.querySelector('.country_filter .selected');
-      if (countrySelect) fireEvent(countrySelect, new MouseEvent('click', { bubbles: true }));
+      if (countrySelect) fireEvent.click(countrySelect);
       expect(document.querySelector('.country_filter .select_list')).toBeVisible();
       const firstCountry = document.querySelector('.country_filter .select_list ul li:nth-child(1)');
-      if (firstCountry) fireEvent(firstCountry, new MouseEvent('click', { bubbles: true }));
+      if (firstCountry) fireEvent.click(firstCountry);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(0);
    });
 
    // Tags Filter should function properly
    it('Tags Filter should Render & Function properly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=filter_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('filter_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.tags_filter')).toBeVisible();
 
       const countrySelect = document.querySelector('.tags_filter .selected');
-      if (countrySelect) fireEvent(countrySelect, new MouseEvent('click', { bubbles: true }));
+      if (countrySelect) fireEvent.click(countrySelect);
       expect(document.querySelector('.tags_filter .select_list')).toBeVisible();
       expect(document.querySelectorAll('.tags_filter .select_list ul li').length).toBe(1);
 
       const firstTag = document.querySelector('.tags_filter .select_list ul li:nth-child(1)');
-      if (firstTag) fireEvent(firstTag, new MouseEvent('click', { bubbles: true }));
+      if (firstTag) fireEvent.click(firstTag);
       expect(document.querySelectorAll('.keyword').length).toBe(1);
    });
 
    it('Sort Options Should be visible Sort Button on Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.sort_options')).toBeVisible();
    });
 
    it('Sort: Position should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
-
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(1)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(2)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });
 
    it('Sort: Date Added should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
 
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(3)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(4)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });
 
    it('Sort: Alphabetical should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
 
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(5)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(6)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });

__tests__/pages/domains.test.tsx

@@ -0,0 +1,49 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from 'react-query';
+import * as ReactQuery from 'react-query';
+import { dummyDomain } from '../../__mocks__/data';
+import Domains from '../../pages/domains';
+
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+jest.spyOn(ReactQuery, 'useQuery').mockImplementation(jest.fn().mockReturnValue(
+   { data: { domains: [dummyDomain] }, isLoading: false, isSuccess: true },
+));
+
+fetchMock.mockIf(`${window.location.origin}/api/domains`, async () => {
+   return new Promise((resolve) => {
+      resolve({
+         body: JSON.stringify({ domains: [dummyDomain] }),
+         status: 200,
+      });
+   });
+});
+
+describe('Domains Page', () => {
+   const queryClient = new QueryClient();
+   it('Renders without crashing', async () => {
+      render(
+          <QueryClientProvider client={queryClient}>
+              <Domains />
+          </QueryClientProvider>,
+      );
+      expect(screen.getByTestId('domains')).toBeInTheDocument();
+   });
+   it('Renders the Domain Component', async () => {
+      const { container } = render(
+          <QueryClientProvider client={queryClient}>
+              <Domains />
+          </QueryClientProvider>,
+      );
+      expect(container.querySelector('.domItem')).toBeInTheDocument();
+   });
+   it('Should Display Add Domain Modal on relveant Button Click.', async () => {
+      render(<QueryClientProvider client={queryClient}><Domains /></QueryClientProvider>);
+      const button = screen.getByTestId('addDomainButton');
+      if (button) fireEvent.click(button);
+      expect(screen.getByTestId('adddomain_modal')).toBeVisible();
+   });
+   it('Should Display the version number in Footer.', async () => {
+      render(<QueryClientProvider client={queryClient}><Domains /></QueryClientProvider>);
+      expect(screen.getByText('SerpBear v0.0.0')).toBeVisible();
+   });
+});

__tests__/pages/index.test.tsx

@@ -2,21 +2,16 @@ import { render, screen } from '@testing-library/react';
 import { QueryClient, QueryClientProvider } from 'react-query';
 import Home from '../../pages/index';
 
+const routerPush = jest.fn();
+jest.mock('next/router', () => ({
+   useRouter: () => ({
+      push: routerPush,
+   }),
+}));
+
 describe('Home Page', () => {
    const queryClient = new QueryClient();
    it('Renders without crashing', async () => {
-      // const dummyDomain = {
-      //    ID: 1,
-      //    domain: 'compressimage.io',
-      //    slug: 'compressimage-io',
-      //    keywordCount: 0,
-      //    lastUpdated: '2022-11-11T10:00:32.243',
-      //    added: '2022-11-11T10:00:32.244',
-      //    tags: [],
-      //    notification: true,
-      //    notification_interval: 'daily',
-      //    notification_emails: '',
-      // };
       render(
           <QueryClientProvider client={queryClient}>
               <Home />
@@ -26,12 +21,12 @@ describe('Home Page', () => {
       expect(await screen.findByRole('main')).toBeInTheDocument();
       expect(screen.queryByText('Add Domain')).not.toBeInTheDocument();
    });
-   it('Should Display the Add Domain Modal when there are no Domains.', async () => {
+   it('Should redirect to /domains route.', async () => {
        render(
            <QueryClientProvider client={queryClient}>
                <Home />
            </QueryClientProvider>,
        );
-       expect(await screen.findByText('Add Domain')).toBeInTheDocument();
+       expect(routerPush).toHaveBeenCalledWith('/domains');
    });
 });

components/common/Icon.tsx

@@ -221,6 +221,14 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
                   </g>
                </svg>
          }
+         {type === 'eye-closed'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <g fill="none" stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth="2">
+                     <path d="M6.873 17.129c-1.845-1.31-3.305-3.014-4.13-4.09a1.693 1.693 0 0 1 0-2.077C4.236 9.013 7.818 5 12 5c1.876 0 3.63.807 5.13 1.874"/>
+                     <path d="M14.13 9.887a3 3 0 1 0-4.243 4.242M4 20L20 4M10 18.704A7.124 7.124 0 0 0 12 19c4.182 0 7.764-4.013 9.257-5.962a1.694 1.694 0 0 0-.001-2.078A22.939 22.939 0 0 0 19.57 9"/>
+                  </g>
+               </svg>
+         }
          {type === 'target'
             && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
                   <path d="M19.938 13A8.004 8.004 0 0 1 13 19.938V22h-2v-2.062A8.004 8.004 0 0 1 4.062 13H2v-2h2.062A8.004 8.004 0 0 1 11 4.062V2h2v2.062A8.004 8.004 0 0 1 19.938 11H22v2h-2.062zM12 18a6 6 0 1 0 0-12a6 6 0 0 0 0 12zm0-3a3 3 0 1 0 0-6a3 3 0 0 0 0 6z" fill={color} fillRule="nonzero"/>
@@ -245,6 +253,27 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
                   <path d="M15.75 9h3v2.25h-3z" fill={color} />
                </svg>
          }
+         {type === 'email'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill={color} d="M22 6c0-1.1-.9-2-2-2H4c-1.1 0-2 .9-2 2v12c0 1.1.9 2 2 2h16c1.1 0 2-.9 2-2zm-2 0l-8 5l-8-5zm0 12H4V8l8 5l8-5z" />
+               </svg>
+         }
+         {type === 'scraper'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 16 16">
+               <path fill={color} d="M1 3.5A2.5 2.5 0 0 1 3.5 1h7A2.5 2.5 0 0 1 13 3.5v1.53a4.538 4.538 0 0 0-1-.004V5H2v5.5A1.5 1.5 0 0 0 3.5 12h2.954l-.72.72a2.52 2.52 0 0 0-.242.28H3.5A2.5 2.5 0 0 1 1 10.5zm7.931 3.224l-.577-.578a.5.5 0 1 0-.708.708l.745.744c.144-.306.324-.6.54-.874M2 4h10v-.5A1.5 1.5 0 0 0 10.5 2h-7A1.5 1.5 0 0 0 2 3.5zm4.354 2.854a.5.5 0 1 0-.708-.708l-2 2a.5.5 0 0 0 0 .708l2 2a.5.5 0 0 0 .708-.708L4.707 8.5zm6.538-.83c.366.042.471.48.21.742l-.975.975a1.507 1.507 0 1 0 2.132 2.132l.975-.975c.261-.261.7-.156.742.21a3.518 3.518 0 0 1-4.676 3.723l-2.726 2.727a1.507 1.507 0 1 1-2.132-2.132L9.168 10.7a3.518 3.518 0 0 1 3.724-4.676" />
+            </svg>
+         }
+         {type === 'city'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 48 48">
+                  <g fill="none">
+                     <path stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth={4} d="M4 42h40"></path>
+                     <rect width={8} height={16} x={8} y={26} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path stroke={color} strokeLinecap="square" strokeLinejoin="round" strokeWidth={4} d="M12 34h1"></path>
+                     <rect width={24} height={38} x={16} y={4} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path fill={color} d="M22 10h4v4h-4zm8 0h4v4h-4zm-8 7h4v4h-4zm8 0h4v4h-4zm0 7h4v4h-4zm0 7h4v4h-4z"></path>
+                  </g>
+               </svg>
+         }
        </span>
    );
  };

components/common/InputField.tsx

@@ -0,0 +1,28 @@
+type InputFieldProps = {
+   label: string;
+   value: string;
+   onChange: Function;
+   placeholder?: string;
+   classNames?: string;
+   hasError?: boolean;
+}
+
+const InputField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: InputFieldProps) => {
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+   return (
+      <div className="field--input w-full relative flex justify-between items-center">
+         <label className={labelStyle}>{label}</label>
+         <input
+            className={`p-2 border border-gray-200 rounded focus:outline-none w-[210px]
+             focus:border-blue-200 ${hasError ? ' border-red-400 focus:border-red-400' : ''} `}
+            type={'text'}
+            value={value}
+            onChange={(event) => onChange(event.target.value)}
+            autoComplete="off"
+            placeholder={placeholder}
+         />
+      </div>
+   );
+};
+
+export default InputField;

components/common/Modal.tsx

@@ -1,25 +1,17 @@
-import React, { useEffect } from 'react';
+import React from 'react';
 import Icon from './Icon';
+import useOnKey from '../../hooks/useOnKey';
 
 type ModalProps = {
    children: React.ReactNode,
    width?: string,
    title?: string,
+   verticalCenter?: boolean,
    closeModal: Function,
 }
 
-const Modal = ({ children, width = '1/2', closeModal, title }:ModalProps) => {
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            closeModal();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeModal]);
+const Modal = ({ children, width = '1/2', closeModal, title, verticalCenter = false }:ModalProps) => {
+   useOnKey('Escape', closeModal);
 
    const closeOnBGClick = (e:React.SyntheticEvent) => {
       e.stopPropagation();
@@ -30,8 +22,9 @@ const Modal = ({ children, width = '1/2', closeModal, title }:ModalProps) => {
    return (
       <div className='modal fixed top-0 left-0 bg-white/[.7] w-full h-screen z-50' onClick={closeOnBGClick}>
          <div
-         className={`modal__content max-w-[340px] absolute top-1/4 left-0 right-0 ml-auto mr-auto w-${width} 
-         lg:max-w-md bg-white shadow-md rounded-md p-5 border-t-[1px] border-gray-100 text-base`}>
+         className={`modal__content max-w-[340px] absolute left-0 right-0 ml-auto mr-auto w-${width} 
+         lg:max-w-md bg-white shadow-md rounded-md p-5 border-t-[1px] border-gray-100 text-base 
+         ${verticalCenter ? ' top-1/2 translate-y-[-50%]' : 'top-1/4'}`}>
             {title && <h3 className=' font-semibold mb-3'>{title}</h3>}
             <button
             className='modal-close absolute right-2 top-2 p-2 cursor-pointer text-gray-400 transition-all

components/common/SecretField.tsx

@@ -0,0 +1,37 @@
+import { useState } from 'react';
+import Icon from './Icon';
+
+type SecretFieldProps = {
+   label: string;
+   value: string;
+   onChange: Function;
+   placeholder?: string;
+   classNames?: string;
+   hasError?: boolean;
+}
+
+const SecretField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: SecretFieldProps) => {
+   const [showValue, setShowValue] = useState(false);
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+   return (
+      <div className="settings__section__secret mb-5 relative flex justify-between items-center">
+         <label className={labelStyle}>{label}</label>
+         <span
+         className="absolute top-1 right-0 px-2 py-1 cursor-pointer text-gray-400 select-none"
+         onClick={() => setShowValue(!showValue)}>
+            <Icon type={showValue ? 'eye-closed' : 'eye'} size={18} />
+         </span>
+         <input
+            className={`w-[210px] p-2 border border-gray-200 rounded focus:outline-none 
+             focus:border-blue-200 ${hasError ? ' border-red-400 focus:border-red-400' : ''} `}
+            type={showValue ? 'text' : 'password'}
+            value={value}
+            onChange={(event) => onChange(event.target.value)}
+            autoComplete="off"
+            placeholder={placeholder}
+         />
+      </div>
+   );
+};
+
+export default SecretField;

components/common/SelectField.tsx

@@ -9,6 +9,7 @@ type SelectFieldProps = {
    defaultLabel: string,
    options: SelectionOption[],
    selected: string[],
+   label?: string,
    multiple?: boolean,
    updateField: Function,
    minWidth?: number,
@@ -28,6 +29,7 @@ const SelectField = (props: SelectFieldProps) => {
       maxHeight = 96,
       rounded = 'rounded-3xl',
       flags = false,
+      label = '',
       emptyMsg = '' } = props;
 
    const [showOptions, setShowOptions] = useState<boolean>(false);
@@ -66,12 +68,13 @@ const SelectField = (props: SelectFieldProps) => {
    };
 
    return (
-       <div className="select font-semibold text-gray-500">
+       <div className="select font-semibold text-gray-500 relative flex justify-between items-center">
+         {label && <label className='mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'>{label}</label>}
          <div
-         className={`selected flex border ${rounded} p-1.5 px-4 cursor-pointer select-none w-[180px] min-w-[${minWidth}px] 
+         className={`selected flex border ${rounded} p-1.5 px-4 cursor-pointer select-none w-[210px] min-w-[${minWidth}px] 
          ${showOptions ? 'border-indigo-200' : ''}`}
          onClick={() => setShowOptions(!showOptions)}>
-            <span className={`w-[${minWidth - 30}px] inline-block truncate mr-2 capitalize`}>
+            <span className={'w-full inline-block truncate mr-2 capitalize'}>
                {selected.length > 0 ? (selectedLabels.slice(0, 2).join(', ')) : defaultLabel}
             </span>
             {multiple && selected.length > 2
@@ -80,7 +83,7 @@ const SelectField = (props: SelectFieldProps) => {
          </div>
          {showOptions && (
             <div
-            className={`select_list mt-1 border absolute min-w-[${minWidth}px] 
+            className={`select_list mt-1 border absolute min-w-[${minWidth}px] top-[30px] right-0 w-[210px]
             ${rounded === 'rounded-3xl' ? 'rounded-lg' : rounded} max-h-${maxHeight} bg-white z-50 overflow-y-auto styled-scrollbar`}>
                {options.length > 20 && (
                   <div className=''>

components/common/Sidebar.tsx

@@ -1,3 +1,4 @@
+/* eslint-disable @next/next/no-img-element */
 import React from 'react';
 import Link from 'next/link';
 import { useRouter } from 'next/router';
@@ -26,11 +27,11 @@ const Sidebar = ({ domains, showAddModal } : SidebarProps) => {
                                         rounded-r-none ${((`/domain/${d.slug}` === router.asPath || `/domain/console/${d.slug}` === router.asPath
                                         || `/domain/insight/${d.slug}` === router.asPath)
                                         ? 'bg-white text-zinc-800 border border-r-0' : 'text-zinc-500')}`}>
-                                          <i className={'text-center leading-4 mr-2 inline-block rounded-full w-5 h-5 bg-orange-200 not-italic'}>
-                                             {d.domain.charAt(0)}
-                                          </i>
+                                          <img
+                                          className={' inline-block mr-1'}
+                                          src={`https://www.google.com/s2/favicons?domain=${d.domain}&sz=16`} alt={d.domain}
+                                          />
                                           {d.domain}
-                                          {/* <span>0</span> */}
                                        </a>
                                     </Link>
                                  </li>)

components/common/ToggleField.tsx

@@ -0,0 +1,32 @@
+type ToggleFieldProps = {
+   label: string;
+   value: string;
+   onChange: (bool:boolean) => void ;
+   classNames?: string;
+}
+
+const ToggleField = ({ label = '', value = '', onChange, classNames = '' }: ToggleFieldProps) => {
+   return (
+      <div className={`field--toggle w-full relative ${classNames}`}>
+         <label className="relative inline-flex items-center cursor-pointer w-full justify-between">
+            <span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-56">{label}</span>
+            <input
+            type="checkbox"
+            value={value}
+            checked={!!value}
+            className="sr-only peer"
+            onChange={() => onChange(!value)}
+            />
+            <div className="relative rounded-3xl w-9 h-5 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
+            peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
+            peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
+            after:absolute after:top-[2px] after:left-[2px] after:bg-white  after:border-gray-300
+            after:border after:rounded-full after:h-4 after:w-4
+            after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
+
+         </label>
+      </div>
+   );
+};
+
+export default ToggleField;

components/domains/AddDomain.tsx

@@ -1,54 +1,66 @@
 import React, { useState } from 'react';
 import Modal from '../common/Modal';
 import { useAddDomain } from '../../services/domains';
-import { isValidDomain } from '../../utils/validators';
+import { isValidUrl } from '../../utils/client/validators';
 
 type AddDomainProps = {
+   domains: DomainType[],
    closeModal: Function
 }
 
-const AddDomain = ({ closeModal }: AddDomainProps) => {
+const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
    const [newDomain, setNewDomain] = useState<string>('');
-   const [newDomainError, setNewDomainError] = useState<boolean>(false);
+   const [newDomainError, setNewDomainError] = useState('');
    const { mutate: addMutate, isLoading: isAdding } = useAddDomain(() => closeModal());
 
    const addDomain = () => {
-      // console.log('ADD NEW DOMAIN', newDomain);
-      if (isValidDomain(newDomain.trim())) {
-         setNewDomainError(false);
-         // TODO: Domain Action
-         addMutate(newDomain.trim());
-      } else {
-         setNewDomainError(true);
+      setNewDomainError('');
+      const existingDomains = domains.map((d) => d.domain);
+      const insertedURLs = newDomain.split('\n');
+      const domainsTobeAdded:string[] = [];
+      const invalidDomains:string[] = [];
+      insertedURLs.forEach((url) => {
+        const theURL = url.trim();
+        if (isValidUrl(theURL)) {
+         const domURL = new URL(theURL);
+         const isDomain = domURL.pathname === '/';
+         if (isDomain && !existingDomains.includes(domURL.host)) {
+            domainsTobeAdded.push(domURL.host);
+         }
+         if (!isDomain && !existingDomains.includes(domURL.href)) {
+            const cleanedURL = domURL.href.replace('https://', '').replace('http://', '').replace(/^\/+|\/+$/g, '');
+            domainsTobeAdded.push(cleanedURL);
+         }
+        } else {
+         invalidDomains.push(theURL);
+        }
+      });
+      if (invalidDomains.length > 0) {
+         setNewDomainError(`Please Insert Valid Domain URL. Invalid URLs: ${invalidDomains.join(', ')}`);
+      } else if (domainsTobeAdded.length > 0) {
+            console.log('domainsTobeAdded :', domainsTobeAdded);
+         addMutate(domainsTobeAdded);
       }
    };
 
-   const handleDomainInput = (e:React.FormEvent<HTMLInputElement>) => {
-      if (e.currentTarget.value === '' && newDomainError) { setNewDomainError(false); }
+   const handleDomainInput = (e:React.ChangeEvent<HTMLTextAreaElement>) => {
+      if (e.currentTarget.value === '' && newDomainError) { setNewDomainError(''); }
       setNewDomain(e.currentTarget.value);
    };
 
    return (
       <Modal closeModal={() => { closeModal(false); }} title={'Add New Domain'}>
          <div data-testid="adddomain_modal">
-            <h4 className='text-sm mt-4'>
-               Domain Name {newDomainError && <span className=' ml-2 block float-right text-red-500 text-xs font-semibold'>Not a Valid Domain</span>}
-            </h4>
-            <input
-               className={`w-full p-2 border border-gray-200 rounded mt-2 mb-3 focus:outline-none  focus:border-blue-200 
-               ${newDomainError ? ' border-red-400 focus:border-red-400' : ''} `}
-               type="text"
+            <h4 className='text-sm mt-4'>Domain URL</h4>
+            <textarea
+               className={`w-full h-40 border rounded border-gray-200 p-4 outline-none
+                focus:border-indigo-300 ${newDomainError ? ' border-red-400 focus:border-red-400' : ''}`}
+               placeholder="Type or Paste URLs here. Insert Each URL in a New line."
                value={newDomain}
-               placeholder={'example.com'}
-               onChange={handleDomainInput}
                autoFocus={true}
-               onKeyDown={(e) => {
-                  if (e.code === 'Enter') {
-                    e.preventDefault();
-                    addDomain();
-                  }
-               }}
-            />
+               onChange={handleDomainInput}>
+            </textarea>
+            {newDomainError && <div><span className=' ml-2 block float-right text-red-500 text-xs font-semibold'>{newDomainError}</span></div>}
             <div className='mt-6 text-right text-sm font-semibold'>
                <button className='py-2 px-5 rounded cursor-pointer bg-indigo-50 text-slate-500 mr-3' onClick={() => closeModal(false)}>Cancel</button>
                <button className='py-2 px-5 rounded cursor-pointer bg-blue-700 text-white' onClick={() => !isAdding && addDomain() }>

components/domains/DomainItem.tsx

@@ -11,9 +11,10 @@ type DomainItemProps = {
    selected: boolean,
    isConsoleIntegrated: boolean,
    thumb: string,
+   updateThumb: Function,
 }
 
-const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: DomainItemProps) => {
+const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb, updateThumb }: DomainItemProps) => {
    const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain;
    // const router = useRouter();
    return (
@@ -21,11 +22,23 @@ const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: Do
          <Link href={`/domain/${slug}`} passHref={true}>
          <a className='flex flex-col lg:flex-row'>
             <div className={`flex-1 p-6 flex ${!isConsoleIntegrated ? 'basis-1/3' : ''}`}>
-               <div className="domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded border border-gray-200 overflow-hidden">
-                  {thumb && <img src={thumb} alt={domain.domain} />}
+               <div className="group domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded
+                  border border-gray-200 overflow-hidden flex justify-center relative">
+                  <button
+                     className=' absolute right-1 top-0 text-gray-400 p-1 transition-all
+                     invisible opacity-0 group-hover:visible group-hover:opacity-100 hover:text-gray-600 z-10'
+                     title='Reload Website Screenshot'
+                     onClick={(e) => { e.preventDefault(); e.stopPropagation(); updateThumb(domain.domain); }}
+                  >
+                     <Icon type="reload" size={12} />
+                  </button>
+                  <img
+                  className={`self-center ${!thumb ? 'max-w-[50px]' : ''}`}
+                  src={thumb || `https://www.google.com/s2/favicons?domain=${domain.domain}&sz=128`} alt={domain.domain}
+                  />
                </div>
                <div className="domain_details flex-1">
-                  <h3 className='font-semibold text-base mb-2'>{domain.domain}</h3>
+                  <h3 className='font-semibold text-base mb-2 max-w-[200px] text-ellipsis overflow-hidden' title={domain.domain}>{domain.domain}</h3>
                  {keywordsUpdated && (
                   <span className=' text-gray-600 text-xs'>
                      Updated <TimeAgo title={dayjs(keywordsUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={keywordsUpdated} />

components/domains/DomainSettings.tsx

@@ -1,8 +1,10 @@
 import { useRouter } from 'next/router';
-import { useEffect, useState } from 'react';
+import { useState } from 'react';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
-import { useDeleteDomain, useUpdateDomain } from '../../services/domains';
+import { useDeleteDomain, useFetchDomain, useUpdateDomain } from '../../services/domains';
+import InputField from '../common/InputField';
+import SelectField from '../common/SelectField';
 
 type DomainSettingsProps = {
    domain:DomainType|false,
@@ -16,28 +18,27 @@ type DomainSettingsError = {
 
 const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
    const router = useRouter();
+   const [currentTab, setCurrentTab] = useState<'notification'|'searchconsole'>('notification');
    const [showRemoveDomain, setShowRemoveDomain] = useState<boolean>(false);
    const [settingsError, setSettingsError] = useState<DomainSettingsError>({ type: '', msg: '' });
-   const [domainSettings, setDomainSettings] = useState<DomainSettings>({ notification_interval: 'never', notification_emails: '' });
+   const [domainSettings, setDomainSettings] = useState<DomainSettings>(() => ({
+      notification_interval: domain && domain.notification_interval ? domain.notification_interval : 'never',
+      notification_emails: domain && domain.notification_emails ? domain.notification_emails : '',
+      search_console: domain && domain.search_console ? JSON.parse(domain.search_console) : {
+         property_type: 'domain', url: '', client_email: '', private_key: '',
+      },
+   }));
 
-   const { mutate: updateMutate } = useUpdateDomain(() => closeModal(false));
-   const { mutate: deleteMutate } = useDeleteDomain(() => {
-      closeModal(false);
-      router.push('/domains');
+   const { mutate: updateMutate, error: domainUpdateError, isLoading: isUpdating } = useUpdateDomain(() => closeModal(false));
+   const { mutate: deleteMutate } = useDeleteDomain(() => { closeModal(false); router.push('/domains'); });
+
+   // Get the Full Domain Data along with the Search Console API Data.
+   useFetchDomain(router, domain && domain.domain ? domain.domain : '', (domainObj:DomainType) => {
+      const currentSearchConsoleSettings = domainObj.search_console && JSON.parse(domainObj.search_console);
+      setDomainSettings({ ...domainSettings, search_console: currentSearchConsoleSettings || domainSettings.search_console });
    });
 
-   useEffect(() => {
-      if (domain) {
-         setDomainSettings({ notification_interval: domain.notification_interval, notification_emails: domain.notification_emails });
-      }
-   }, [domain]);
-
-   const updateNotiEmails = (event:React.FormEvent<HTMLInputElement>) => {
-      setDomainSettings({ ...domainSettings, notification_emails: event.currentTarget.value });
-   };
-
    const updateDomain = () => {
-      console.log('Domain: ');
       let error: DomainSettingsError | null = null;
       if (domainSettings.notification_emails) {
          const notification_emails = domainSettings.notification_emails.split(',');
@@ -58,24 +59,103 @@ const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
          }
    };
 
+   const tabStyle = `inline-block px-4 py-2 rounded-md mr-3 cursor-pointer text-sm select-none z-10
+                     text-gray-600 border border-b-0 relative top-[1px] rounded-b-none`;
    return (
       <div>
-         <Modal closeModal={() => closeModal(false)} title={'Domain Settings'} width="[500px]">
+         <Modal closeModal={() => closeModal(false)} title={'Domain Settings'} width="[500px]" verticalCenter={currentTab === 'searchconsole'} >
             <div data-testid="domain_settings" className=" text-sm">
-               <div className="mb-6 flex justify-between items-center">
-                  <h4>Notification Emails
-                     {settingsError.type === 'email' && <span className="text-red-500 font-semibold ml-2">{settingsError.msg}</span>}
-                  </h4>
-                  <input
-                     className={`border w-46 text-sm transition-all rounded p-1.5 px-4 outline-none ring-0 
-                     ${settingsError.type === 'email' ? ' border-red-300' : ''}`}
-                     type="text"
-                     placeholder='Your Emails'
-                     onChange={updateNotiEmails}
-                     value={domainSettings.notification_emails || ''}
-                  />
+               <div className=' mt-3 mb-5 border  border-slate-200 px-2 py-4 pb-0
+               relative left-[-20px] w-[calc(100%+40px)] border-l-0 border-r-0 bg-[#f8f9ff]'>
+                  <ul>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'notification' ? ' bg-white text-blue-600 border-slate-200' : 'border-transparent'} `}
+                     onClick={() => setCurrentTab('notification')}>
+                       <Icon type='email' /> Notification
+                     </li>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'searchconsole' ? ' bg-white text-blue-600 border-slate-200' : 'border-transparent'}`}
+                     onClick={() => setCurrentTab('searchconsole')}>
+                        <Icon type='google' /> Search Console
+                     </li>
+                  </ul>
                </div>
+
+               <div>
+                  {currentTab === 'notification' && (
+                     <div className="mb-4 flex justify-between items-center w-full">
+                        <InputField
+                        label='Notification Emails'
+                        onChange={(emails:string) => setDomainSettings({ ...domainSettings, notification_emails: emails })}
+                        value={domainSettings.notification_emails || ''}
+                        placeholder='Your Emails'
+                        />
+                     </div>
+                  )}
+                  {currentTab === 'searchconsole' && (
+                     <>
+                        <div className="mb-4 flex justify-between items-center w-full">
+                           <label className='mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'>Property Type</label>
+                           <SelectField
+                           options={[{ label: 'Domain', value: 'domain' }, { label: 'URL', value: 'url' }]}
+                           selected={[domainSettings.search_console?.property_type || 'domain']}
+                           defaultLabel="Select Search Console Property Type"
+                           updateField={(updated:['domain'|'url']) => setDomainSettings({
+                              ...domainSettings,
+                              search_console: { ...(domainSettings.search_console as DomainSearchConsole), property_type: updated[0] || 'domain' },
+                           })}
+                           multiple={false}
+                           rounded={'rounded'}
+                           />
+                        </div>
+                        {domainSettings?.search_console?.property_type === 'url' && (
+                           <div className="mb-4 flex justify-between items-center w-full">
+                              <InputField
+                              label='Property URL (Required)'
+                              onChange={(url:string) => setDomainSettings({
+                                 ...domainSettings,
+                                 search_console: { ...(domainSettings.search_console as DomainSearchConsole), url },
+                              })}
+                              value={domainSettings?.search_console?.url || ''}
+                              placeholder='Search Console Property URL. eg: https://mywebsite.com/'
+                              />
+                           </div>
+                        )}
+                        <div className="mb-4 flex justify-between items-center w-full">
+                           <InputField
+                           label='Search Console Client Email'
+                           onChange={(client_email:string) => setDomainSettings({
+                              ...domainSettings,
+                              search_console: { ...(domainSettings.search_console as DomainSearchConsole), client_email },
+                           })}
+                           value={domainSettings?.search_console?.client_email || ''}
+                           placeholder='myapp@appspot.gserviceaccount.com'
+                           />
+                        </div>
+                        <div className="mb-4 flex flex-col justify-between items-center w-full">
+                           <label className='mb-2 font-semibold block text-sm text-gray-700 capitalize w-full'>Search Console Private Key</label>
+                           <textarea
+                              className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
+                              focus:outline-none h-[100px] focus:border-blue-200`}
+                              value={domainSettings?.search_console?.private_key || ''}
+                              placeholder={'-----BEGIN PRIVATE KEY-----/ssssaswdkihad....'}
+                              onChange={(event) => setDomainSettings({
+                                 ...domainSettings,
+                                 search_console: { ...(domainSettings.search_console as DomainSearchConsole), private_key: event.target.value },
+                              })}
+                           />
+                        </div>
+                     </>
+                  )}
+               </div>
+               {!isUpdating && (domainUpdateError as Error)?.message && (
+                  <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{(domainUpdateError as Error).message}</div>
+               )}
+               {!isUpdating && settingsError?.msg && (
+                  <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{settingsError.msg}</div>
+               )}
             </div>
+
             <div className="flex justify-between border-t-[1px] border-gray-100 mt-8 pt-4 pb-0">
                <button
                className="text-sm font-semibold text-red-500"
@@ -83,9 +163,9 @@ const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
                   <Icon type="trash" /> Remove Domain
                </button>
                <button
-               className='text-sm font-semibold py-2 px-5 rounded cursor-pointer bg-blue-700 text-white'
-               onClick={() => updateDomain()}>
-                  Update Settings
+               className={`text-sm font-semibold py-2 px-5 rounded cursor-pointer bg-blue-700 text-white ${isUpdating ? 'cursor-not-allowed' : ''}`}
+               onClick={() => !isUpdating && updateDomain()}>
+                  {isUpdating && <Icon type='loading' />} Update Settings
                </button>
             </div>
          </Modal>

components/insight/Insight.tsx

@@ -12,7 +12,7 @@ type SCInsightProps = {
    isConsoleIntegrated: boolean,
 }
 
-const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SCInsightProps) => {
+const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true, domain }: SCInsightProps) => {
    const [activeTab, setActiveTab] = useState<string>('stats');
 
    const insightItems = insight[activeTab as keyof InsightDataType];
@@ -108,7 +108,7 @@ const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SC
                               (item:SCInsightItem, index: number) => {
                               const insightItemCount = insight ? insightItems : [];
                               const lastItem = !!(insightItemCount && (index === insightItemCount.length));
-                              return <InsightItem key={index} item={item} type={activeTab} lastItem={lastItem} />;
+                              return <InsightItem key={index} item={item} type={activeTab} lastItem={lastItem} domain={domain?.domain || ''} />;
                            },
                         )
                      }
@@ -117,7 +117,7 @@ const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SC
                      )}
                      {!isConsoleIntegrated && (
                         <p className=' p-9 pt-[10%] text-center text-gray-500'>
-                        Google Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        Google Search Console has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
                         </p>
                      )}
                   </div>

components/insight/InsightItem.tsx

@@ -1,21 +1,22 @@
 import React from 'react';
 import countries from '../../utils/countries';
 import Icon from '../common/Icon';
+import { formattedNum } from '../../utils/client/helpers';
 
 type InsightItemProps = {
    item: SCInsightItem,
    lastItem: boolean,
-   type: string
+   type: string,
+   domain: string
 }
 
-const InsightItem = ({ item, lastItem, type }:InsightItemProps) => {
+const InsightItem = ({ item, lastItem, type, domain }:InsightItemProps) => {
    const { clicks, impressions, ctr, position, country = 'zzz', keyword, page, keywords = 0, countries: cntrs = 0, date } = item;
    let firstItem = keyword;
    if (type === 'pages') { firstItem = page; } if (type === 'stats') {
       firstItem = date && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(date));
    }
    if (type === 'countries') { firstItem = countries[country] && countries[country][0]; }
-   const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num);
 
    return (
       <div
@@ -24,7 +25,7 @@ const InsightItem = ({ item, lastItem, type }:InsightItemProps) => {
 
          <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold'>
             {type === 'countries' && <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} />}
-            {firstItem}
+            {type === 'pages' && domain ? <a href={`https://${domain}${page}`} target='_blank' rel="noreferrer">{firstItem}</a> : firstItem}
          </div>
 
          <div className='keyword_pos text-center inline-block mr-3 lg:mr-0 lg:flex-1'>
@@ -34,7 +35,6 @@ const InsightItem = ({ item, lastItem, type }:InsightItemProps) => {
             {Math.round(position)}
          </div>
 
-         {/* <div className='keyword_imp text-center inline-block lg:flex-1'>{formattedNum(clicks)}</div> */}
          <div className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-14 p-2 text-base mt-[-55px] rounded right-5 lg:relative
           lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-40 lg:grow-0 lg:right-0 text-center font-semibold`}>
             {formattedNum(clicks)}

components/insight/InsightStats.tsx

@@ -1,6 +1,7 @@
-import React, { useMemo, useState, useEffect } from 'react';
+import React, { useMemo } from 'react';
 import { Chart as ChartJS, CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend } from 'chart.js';
 import { Line } from 'react-chartjs-2';
+import { formattedNum } from '../../utils/client/helpers';
 
 ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend);
 
@@ -12,21 +13,15 @@ type InsightStatsProps = {
 }
 
 const InsightStats = ({ stats = [], totalKeywords = 0, totalPages = 0 }:InsightStatsProps) => {
-   const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num);
-   const [totalStat, setTotalStat] = useState({ impressions: 0, clicks: 0, ctr: 0, position: 0 });
-
-   useEffect(() => {
-      if (stats.length > 0) {
-         const totalStats = stats.reduce((acc, item) => {
-            return {
-               impressions: item.impressions + acc.impressions,
-               clicks: item.clicks + acc.clicks,
-               ctr: item.ctr + acc.ctr,
-               position: item.position + acc.position,
-            };
-         }, { impressions: 0, clicks: 0, ctr: 0, position: 0 });
-         setTotalStat(totalStats);
-      }
+   const totalStat = useMemo(() => {
+      return stats.reduce((acc, item) => {
+         return {
+            impressions: item.impressions + acc.impressions,
+            clicks: item.clicks + acc.clicks,
+            ctr: item.ctr + acc.ctr,
+            position: item.position + acc.position,
+         };
+      }, { impressions: 0, clicks: 0, ctr: 0, position: 0 });
    }, [stats]);
 
    const chartData = useMemo(() => {

components/keywords/AddKeywords.tsx

@@ -7,6 +7,8 @@ import { useAddKeywords } from '../../services/keywords';
 
 type AddKeywordsProps = {
    keywords: KeywordType[],
+   scraperName: string,
+   allowsCity: boolean,
    closeModal: Function,
    domain: string
 }
@@ -17,9 +19,10 @@ type KeywordsInput = {
    country: string,
    domain: string,
    tags: string,
+   city?:string,
 }
 
-const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
+const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCity = false }: AddKeywordsProps) => {
    const [error, setError] = useState<string>('');
    const defCountry = localStorage.getItem('default_country') || 'US';
    const [newKeywordsData, setNewKeywordsData] = useState<KeywordsInput>({ keywords: '', device: 'desktop', country: defCountry, domain, tags: '' });
@@ -29,14 +32,16 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
    const addKeywords = () => {
       if (newKeywordsData.keywords) {
          const keywordsArray = [...new Set(newKeywordsData.keywords.split('\n').map((item) => item.trim()).filter((item) => !!item))];
-         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}`);
-         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(`${k}-${newKeywordsData.device}-${newKeywordsData.country}`));
+         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}${k.city ? `-${k.city}` : ''}`);
+         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(
+            `${k}-${newKeywordsData.device}-${newKeywordsData.country}${newKeywordsData.city ? `-${newKeywordsData.city}` : ''}`,
+         ));
          if (keywordExist.length > 0) {
             setError(`Keywords ${keywordExist.join(',')} already Exist`);
             setTimeout(() => { setError(''); }, 3000);
          } else {
-            const { device, country, domain: kDomain, tags } = newKeywordsData;
-            const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags }));
+            const { device, country, domain: kDomain, tags, city } = newKeywordsData;
+            const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags, city }));
             addMutate(newKeywordsArray);
          }
       } else {
@@ -85,17 +90,28 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
                         ><Icon type='mobile' /> <i className='not-italic hidden lg:inline-block'>Mobile</i></li>
                   </ul>
                </div>
-
                <div className='relative'>
                   {/* TODO:  Insert Existing Tags as Suggestions */}
                   <input
                      className='w-full border rounded border-gray-200 py-2 px-4 pl-8 outline-none focus:border-indigo-300'
-                     placeholder='Insert Tags'
+                     placeholder='Insert Tags (Optional)'
                      value={newKeywordsData.tags}
                      onChange={(e) => setNewKeywordsData({ ...newKeywordsData, tags: e.target.value })}
                   />
                   <span className='absolute text-gray-400 top-2 left-2'><Icon type="tags" size={16} /></span>
                </div>
+               <div className='relative mt-2'>
+                  <input
+                     className={`w-full border rounded border-gray-200 py-2 px-4 pl-8 
+                     outline-none focus:border-indigo-300 ${!allowsCity ? ' cursor-not-allowed' : ''} `}
+                     disabled={!allowsCity}
+                     title={!allowsCity ? `Your scraper ${scraperName} doesn't have city level scraping feature.` : ''}
+                     placeholder={`City (Optional)${!allowsCity ? `. Not avaialable for ${scraperName}.` : ''}`}
+                     value={newKeywordsData.city}
+                     onChange={(e) => setNewKeywordsData({ ...newKeywordsData, city: e.target.value })}
+                  />
+                  <span className='absolute text-gray-400 top-2 left-2'><Icon type="city" size={16} /></span>
+               </div>
             </div>
             {error && <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{error}</div>}
             <div className='mt-6 text-right text-sm font-semibold flex justify-between'>

components/keywords/Keyword.tsx

@@ -4,7 +4,8 @@ import dayjs from 'dayjs';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
 import ChartSlim from '../common/ChartSlim';
-import { generateTheChartData } from '../common/generateChartData';
+import KeywordPosition from './KeywordPosition';
+import { generateTheChartData } from '../../utils/client/generateChartData';
 
 type KeywordProps = {
    keywordData: KeywordType,
@@ -39,7 +40,7 @@ const Keyword = (props: KeywordProps) => {
       scDataType = 'threeDays',
    } = props;
    const {
-      keyword, domain, ID, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
+      keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
    } = keywordData;
    const [showOptions, setShowOptions] = useState(false);
    const [showPositionError, setPositionError] = useState(false);
@@ -82,24 +83,14 @@ const Keyword = (props: KeywordProps) => {
 
    const optionsButtonStyle = 'block px-2 py-2 cursor-pointer hover:bg-indigo-50 hover:text-blue-700';
 
-   const renderPosition = (pos:number, type?:string) => {
-      if (!updating && pos === 0) {
-         return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
-      }
-      if (updating && type !== 'sc') {
-         return <span title='Updating Keyword Position'><Icon type="loading" /></span>;
-      }
-      return pos;
-   };
-
    return (
       <div
-      key={keyword}
+      key={keyword + ID}
       style={style}
       className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
       lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
 
-         <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold cursor-pointer'>
+         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:basis-20 lg:w-auto lg:flex lg:items-center'>
             <button
                className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
                ${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
@@ -108,9 +99,10 @@ const Keyword = (props: KeywordProps) => {
                   <Icon type="check" size={10} />
             </button>
             <a
-            className='py-2 hover:text-blue-600'
+            className='py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full'
             onClick={() => showKeywordDetails()}>
-               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />{keyword}
+               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />
+               <span className=' text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-30px)]'>{keyword}{city ? ` (${city})` : ''}</span>
             </a>
             {sticky && <button className='ml-2 relative top-[2px]' title='Favorite'><Icon type="star-filled" size={16} color="#fbd346" /></button>}
             {lastUpdateError && lastUpdateError.date
@@ -123,7 +115,7 @@ const Keyword = (props: KeywordProps) => {
          <div
          className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative
           lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center font-semibold`}>
-            {renderPosition(position)}
+            <KeywordPosition position={position} updating={updating} />
             {!updating && positionChange > 0 && <i className=' not-italic ml-1 text-xs text-[#5ed7c3]'>▲ {positionChange}</i>}
             {!updating && positionChange < 0 && <i className=' not-italic ml-1 text-xs text-red-300'>▼ {positionChange}</i>}
          </div>
@@ -164,7 +156,10 @@ const Keyword = (props: KeywordProps) => {
             relative flex justify-between text-center lg:flex-1 lg:text-sm lg:m-0 lg:mt-0 lg:border-t-0 lg:pt-0 lg:top-0'>
                <span className='min-w-[40px]'>
                   <span className='lg:hidden'>SC Position: </span>
-                  {renderPosition(keywordData?.scData?.position[scDataType as keyof KeywordSCDataChild] || 0, 'sc')}
+                  <KeywordPosition
+                  position={keywordData?.scData?.position[scDataType as keyof KeywordSCDataChild] || 0}
+                  type='sc'
+                  />
                </span>
                <span className='min-w-[40px]'>
                   <span className='lg:hidden'>Impressions: </span>{keywordData?.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0}

components/keywords/KeywordDetails.tsx

@@ -1,10 +1,12 @@
-import React, { useEffect, useMemo, useRef, useState } from 'react';
+import React, { useLayoutEffect, useMemo, useRef, useState } from 'react';
 import dayjs from 'dayjs';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
 import Chart from '../common/Chart';
 import SelectField from '../common/SelectField';
-import { generateTheChartData } from '../common/generateChartData';
+import { useFetchSingleKeyword } from '../../services/keywords';
+import useOnKey from '../../hooks/useOnKey';
+import { generateTheChartData } from '../../utils/client/generateChartData';
 
 type KeywordDetailsProps = {
    keyword: KeywordType,
@@ -13,11 +15,12 @@ type KeywordDetailsProps = {
 
 const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
    const updatedDate = new Date(keyword.lastUpdated);
-   const [keywordHistory, setKeywordHistory] = useState<KeywordHistory>(keyword.history);
-   const [keywordSearchResult, setKeywordSearchResult] = useState<KeywordLastResult[]>([]);
    const [chartTime, setChartTime] = useState<string>('30');
    const searchResultContainer = useRef<HTMLDivElement>(null);
    const searchResultFound = useRef<HTMLDivElement>(null);
+   const { data: keywordData } = useFetchSingleKeyword(keyword.ID);
+   const keywordHistory: KeywordHistory = keywordData?.history || keyword.history;
+   const keywordSearchResult: KeywordLastResult = keywordData?.searchResult || keyword.history;
    const dateOptions = [
       { label: 'Last 7 Days', value: '7' },
       { label: 'Last 30 Days', value: '30' },
@@ -26,39 +29,9 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
       { label: 'All Time', value: 'all' },
    ];
 
-   useEffect(() => {
-      const fetchFullKeyword = async () => {
-         try {
-            const fetchURL = `${window.location.origin}/api/keyword?id=${keyword.ID}`;
-            const res = await fetch(fetchURL, { method: 'GET' }).then((result) => result.json());
-            if (res.keyword) {
-               console.log(res.keyword, new Date().getTime());
-               setKeywordHistory(res.keyword.history || []);
-               setKeywordSearchResult(res.keyword.lastResult || []);
-            }
-         } catch (error) {
-            console.log(error);
-         }
-      };
-      if (keyword.lastResult.length === 0) {
-         fetchFullKeyword();
-      }
-   }, [keyword]);
+   useOnKey('Escape', closeDetails);
 
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            console.log(event.key);
-            closeDetails();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeDetails]);
-
-   useEffect(() => {
+   useLayoutEffect(() => {
       if (keyword.position < 100 && keyword.position > 0 && searchResultFound?.current) {
          searchResultFound.current.scrollIntoView({
             behavior: 'smooth',

components/keywords/KeywordFilter.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useEffect, useMemo } from 'react';
+import React, { useState, useMemo } from 'react';
 import Icon from '../common/Icon';
 import SelectField, { SelectionOption } from '../common/SelectField';
 import countries from '../../utils/countries';
@@ -17,18 +17,13 @@ type KeywordFilterProps = {
    SCcountries?: string[];
 }
 
-type KeywordCountState = {
-   desktop: number,
-   mobile: number
-}
-
 const KeywordFilters = (props: KeywordFilterProps) => {
    const {
       device,
       setDevice,
       filterKeywords,
       allTags = [],
-      keywords,
+      keywords = [],
       updateSort,
       sortBy,
       filterParams,
@@ -36,20 +31,21 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       integratedConsole = false,
       SCcountries = [],
     } = props;
-   const [keywordCounts, setKeywordCounts] = useState<KeywordCountState>({ desktop: 0, mobile: 0 });
    const [sortOptions, showSortOptions] = useState(false);
    const [filterOptions, showFilterOptions] = useState(false);
 
-   useEffect(() => {
-      const keyWordCount = { desktop: 0, mobile: 0 };
-      keywords.forEach((k) => {
-         if (k.device === 'desktop') {
-            keyWordCount.desktop += 1;
-         } else {
-            keyWordCount.mobile += 1;
-         }
-      });
-      setKeywordCounts(keyWordCount);
+   const keywordCounts = useMemo(() => {
+      const counts = { desktop: 0, mobile: 0 };
+      if (keywords && keywords.length > 0) {
+         keywords.forEach((k) => {
+            if (k.device === 'desktop') {
+               counts.desktop += 1;
+            } else {
+               counts.mobile += 1;
+            }
+         });
+      }
+      return counts;
    }, [keywords]);
 
    const filterCountry = (cntrs:string[]) => filterKeywords({ ...filterParams, countries: cntrs });

components/keywords/KeywordPosition.tsx

@@ -0,0 +1,19 @@
+import Icon from '../common/Icon';
+
+type KeywordPositionProps = {
+   position: number,
+   updating?: boolean,
+   type?: string,
+}
+
+const KeywordPosition = ({ position = 0, type = '', updating = false }:KeywordPositionProps) => {
+   if (!updating && position === 0) {
+      return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
+   }
+   if (updating && type !== 'sc') {
+      return <span title='Updating Keyword Position'><Icon type="loading" /></span>;
+   }
+   return <>{Math.round(position)}</>;
+};
+
+export default KeywordPosition;

components/keywords/KeywordsTable.tsx

@@ -1,9 +1,7 @@
-import React, { useState, useMemo, useEffect } from 'react';
+import React, { useState, useMemo } from 'react';
 import { Toaster } from 'react-hot-toast';
-import { CSSTransition } from 'react-transition-group';
 import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
-import AddKeywords from './AddKeywords';
-import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/sortFilter';
+import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/client/sortFilter';
 import Icon from '../common/Icon';
 import Keyword from './Keyword';
 import KeywordDetails from './KeywordDetails';
@@ -12,6 +10,8 @@ import Modal from '../common/Modal';
 import { useDeleteKeywords, useFavKeywords, useRefreshKeywords } from '../../services/keywords';
 import KeywordTagManager from './KeywordTagManager';
 import AddTags from './AddTags';
+import useWindowResize from '../../hooks/useWindowResize';
+import useIsMobile from '../../hooks/useIsMobile';
 
 type KeywordsTableProps = {
    domain: DomainType | null,
@@ -23,7 +23,7 @@ type KeywordsTableProps = {
 }
 
 const KeywordsTable = (props: KeywordsTableProps) => {
-   const { domain, keywords = [], isLoading = true, showAddModal = false, setShowAddModal, isConsoleIntegrated = false } = props;
+   const { keywords = [], isLoading = true, isConsoleIntegrated = false } = props;
    const showSCData = isConsoleIntegrated;
    const [device, setDevice] = useState<string>('desktop');
    const [selectedKeywords, setSelectedKeywords] = useState<number[]>([]);
@@ -31,7 +31,6 @@ const KeywordsTable = (props: KeywordsTableProps) => {
    const [showRemoveModal, setShowRemoveModal] = useState<boolean>(false);
    const [showTagManager, setShowTagManager] = useState<null|number>(null);
    const [showAddTags, setShowAddTags] = useState<boolean>(false);
-   const [isMobile, setIsMobile] = useState<boolean>(false);
    const [SCListHeight, setSCListHeight] = useState(500);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
    const [sortBy, setSortBy] = useState<string>('date_asc');
@@ -40,26 +39,18 @@ const KeywordsTable = (props: KeywordsTableProps) => {
    const { mutate: deleteMutate } = useDeleteKeywords(() => {});
    const { mutate: favoriteMutate } = useFavKeywords(() => {});
    const { mutate: refreshMutate } = useRefreshKeywords(() => {});
+   const [isMobile] = useIsMobile();
+   useWindowResize(() => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400)));
 
    const scDataObject:{ [k:string] : string} = {
       threeDays: 'Last Three Days',
       sevenDays: 'Last Seven Days',
       thirtyDays: 'Last Thirty Days',
-      avgSevenDays: 'Last Three Days Avg',
-      avgThreeDays: 'Last Seven Days Avg',
+      avgThreeDays: 'Last Three Days Avg',
+      avgSevenDays: 'Last Seven Days Avg',
       avgThirtyDays: 'Last Thirty Days Avg',
    };
 
-   useEffect(() => {
-      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
-      const resizeList = () => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400));
-      resizeList();
-      window.addEventListener('resize', resizeList);
-      return () => {
-          window.removeEventListener('resize', resizeList);
-      };
-   }, [isMobile]);
-
    const processedKeywords: {[key:string] : KeywordType[]} = useMemo(() => {
       const procKeywords = keywords.filter((x) => x.device === device);
       const filteredKeywords = filterKeywords(procKeywords, filterParams);
@@ -250,13 +241,6 @@ const KeywordsTable = (props: KeywordsTableProps) => {
                   </div>
             </Modal>
          )}
-         <CSSTransition in={showAddModal} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddKeywords
-               domain={domain?.domain || ''}
-               keywords={keywords}
-               closeModal={() => setShowAddModal(false)}
-               />
-         </CSSTransition>
          {showTagManager && (
             <KeywordTagManager
                allTags={allDomainTags}

components/keywords/SCKeyword.tsx

@@ -1,6 +1,8 @@
 import React from 'react';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
+import KeywordPosition from './KeywordPosition';
+import { formattedNum } from '../../utils/client/helpers';
 
 type SCKeywordProps = {
    keywordData: SearchAnalyticsItem,
@@ -15,13 +17,6 @@ const SCKeyword = (props: SCKeywordProps) => {
    const { keywordData, selected, lastItem, selectKeyword, style, isTracked = false } = props;
    const { keyword, uid, position, country, impressions, ctr, clicks } = keywordData;
 
-   const renderPosition = () => {
-      if (position === 0) {
-         return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
-      }
-      return Math.round(position);
-   };
-
    return (
       <div
       key={keyword}
@@ -45,7 +40,7 @@ const SCKeyword = (props: SCKeywordProps) => {
 
          <div className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-15 p-2 text-base mt-[-20px] rounded right-5 lg:relative
           lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-40 lg:grow-0 lg:right-0 text-center font-semibold`}>
-            {renderPosition()}
+            <KeywordPosition position={position} />
             <span className='block text-xs text-gray-500 lg:hidden'>Position</span>
          </div>
 
@@ -53,14 +48,14 @@ const SCKeyword = (props: SCKeywordProps) => {
             <span className='mr-3 lg:hidden'>
                <Icon type="eye" size={14} color="#999" />
             </span>
-            {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(impressions)}
+            {formattedNum(impressions)}
          </div>
 
          <div className={'keyword_visits text-center inline-block mt-4 mr-5 ml-5 lg:flex-1 lg:m-0 max-w-[70px] lg:max-w-none lg:pr-5'}>
             <span className='mr-3 lg:hidden'>
                <Icon type="cursor" size={14} color="#999" />
             </span>
-            {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(clicks)}
+            {formattedNum(clicks)}
          </div>
 
          <div className='keyword_ctr text-center inline-block mt-4 relative lg:flex-1 lg:m-0 '>

components/keywords/SCKeywordsTable.tsx

@@ -1,12 +1,15 @@
 import { useRouter } from 'next/router';
-import React, { useState, useMemo, useEffect } from 'react';
+import React, { useState, useMemo } from 'react';
 import { Toaster } from 'react-hot-toast';
 import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
 import { useAddKeywords, useFetchKeywords } from '../../services/keywords';
-import { SCfilterKeywords, SCkeywordsByDevice, SCsortKeywords } from '../../utils/SCsortFilter';
+import { SCfilterKeywords, SCkeywordsByDevice, SCsortKeywords } from '../../utils/client/SCsortFilter';
 import Icon from '../common/Icon';
 import KeywordFilters from './KeywordFilter';
 import SCKeyword from './SCKeyword';
+import useWindowResize from '../../hooks/useWindowResize';
+import useIsMobile from '../../hooks/useIsMobile';
+import { formattedNum } from '../../utils/client/helpers';
 
 type SCKeywordsTableProps = {
    domain: DomainType | null,
@@ -27,11 +30,13 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
    const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
    const [sortBy, setSortBy] = useState<string>('imp_desc');
-   const [isMobile, setIsMobile] = useState<boolean>(false);
    const [SCListHeight, setSCListHeight] = useState(500);
-   const { keywordsData } = useFetchKeywords(router);
+   const { keywordsData } = useFetchKeywords(router, domain?.domain || '');
    const addedkeywords: string[] = keywordsData?.keywords?.map((key: KeywordType) => `${key.keyword}:${key.country}:${key.device}`) || [];
    const { mutate: addKeywords } = useAddKeywords(() => { if (domain && domain.slug) router.push(`/domain/${domain.slug}`); });
+   const [isMobile] = useIsMobile();
+   useWindowResize(() => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400)));
+
    const finalKeywords: {[key:string] : SCKeywordType[] } = useMemo(() => {
       const procKeywords = keywords.filter((x) => x.device === device);
       const filteredKeywords = SCfilterKeywords(procKeywords, filterParams);
@@ -71,16 +76,6 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
          };
    }, [finalKeywords, device]);
 
-   useEffect(() => {
-      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
-      const resizeList = () => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400));
-      resizeList();
-      window.addEventListener('resize', resizeList);
-      return () => {
-          window.removeEventListener('resize', resizeList);
-      };
-   }, [isMobile]);
-
    const selectKeyword = (keywordID: string) => {
       console.log('Select Keyword: ', keywordID);
       let updatedSelectd = [...selectedKeywords, keywordID];
@@ -194,10 +189,10 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
                               </span>
                               <span className='domKeywords_head_position flex-1 basis-40 grow-0 text-center'>{viewSummary.position}</span>
                               <span className='domKeywords_head_imp flex-1 text-center'>
-                                 {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(viewSummary.impressions)}
+                                 {formattedNum(viewSummary.impressions)}
                               </span>
                               <span className='domKeywords_head_visits flex-1 text-center'>
-                                 {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(viewSummary.visits)}
+                                 {formattedNum(viewSummary.visits)}
                               </span>
                               <span className='domKeywords_head_ctr flex-1 text-center'>
                                  {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(viewSummary.ctr)}%
@@ -214,7 +209,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
                      )}
                      {!isConsoleIntegrated && (
                         <p className=' p-9 pt-[10%] text-center text-gray-500'>
-                        Google Search has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        Google Search Console has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
                         </p>
                      )}
                   </div>

components/settings/NotificationSettings.tsx

@@ -1,5 +1,7 @@
 import React from 'react';
 import SelectField from '../common/SelectField';
+import SecretField from '../common/SecretField';
+import InputField from '../common/InputField';
 
 type NotificationSettingsProps = {
    settings: SettingsType,
@@ -17,8 +19,8 @@ const NotificationSettings = ({ settings, settingsError, updateSettings }:Notifi
       <div>
          <div className='settings__content styled-scrollbar p-6 text-sm'>
             <div className="settings__section__input mb-5">
-               <label className={labelStyle}>Notification Frequency</label>
                <SelectField
+               label='Notification Frequency'
                   multiple={false}
                   selected={[settings.notification_interval]}
                   options={[
@@ -31,70 +33,61 @@ const NotificationSettings = ({ settings, settingsError, updateSettings }:Notifi
                   updateField={(updated:string[]) => updated[0] && updateSettings('notification_interval', updated[0])}
                   rounded='rounded'
                   maxHeight={48}
-                  minWidth={270}
+                  minWidth={220}
                />
             </div>
             {settings.notification_interval !== 'never' && (
                <>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>Notification Emails</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError?.type === 'no_email' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
-                        value={settings?.notification_email}
-                        placeholder={'test@gmail.com'}
-                        onChange={(event) => updateSettings('notification_email', event.target.value)}
+                     <InputField
+                     label='Notification Emails'
+                     hasError={settingsError?.type === 'no_email'}
+                     value={settings?.notification_email}
+                     placeholder={'test@gmail.com, test2@test.com'}
+                     onChange={(value:string) => updateSettings('notification_email', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Server</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError?.type === 'no_smtp_server' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
-                        value={settings?.smtp_server || ''}
-                        onChange={(event) => updateSettings('smtp_server', event.target.value)}
+                     <InputField
+                     label='SMTP Server'
+                     hasError={settingsError?.type === 'no_smtp_server'}
+                     value={settings?.smtp_server || ''}
+                     placeholder={'test@gmail.com, test2@test.com'}
+                     onChange={(value:string) => updateSettings('smtp_server', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Port</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError && settingsError.type === 'no_smtp_port' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
-                        value={settings?.smtp_port || ''}
-                        onChange={(event) => updateSettings('smtp_port', event.target.value)}
+                     <InputField
+                     label='SMTP Port'
+                     hasError={settingsError?.type === 'no_smtp_port'}
+                     value={settings?.smtp_port || ''}
+                     placeholder={'2234'}
+                     onChange={(value:string) => updateSettings('smtp_port', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Username</label>
-                     <input
-                        className={'w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200'}
-                        type="text"
+                     <InputField
+                        label='SMTP Username'
+                        hasError={settingsError?.type === 'no_smtp_port'}
                         value={settings?.smtp_username || ''}
-                        onChange={(event) => updateSettings('smtp_username', event.target.value)}
+                        onChange={(value:string) => updateSettings('smtp_username', value)}
+                        />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                     <SecretField
+                     label='SMTP Password'
+                     value={settings?.smtp_password || ''}
+                     onChange={(value:string) => updateSettings('smtp_password', value)}
                      />
                   </div>
                   <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>SMTP Password</label>
-                     <input
-                        className={'w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200'}
-                        type="text"
-                        value={settings?.smtp_password || ''}
-                        onChange={(event) => updateSettings('smtp_password', event.target.value)}
-                     />
-                  </div>
-                  <div className="settings__section__input mb-5">
-                     <label className={labelStyle}>From Email Address</label>
-                     <input
-                        className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                        ${settingsError?.type === 'no_smtp_from' ? ' border-red-400 focus:border-red-400' : ''} `}
-                        type="text"
+                        <InputField
+                        label='From Email Address'
+                        hasError={settingsError?.type === 'no_smtp_from'}
                         value={settings?.notification_email_from || ''}
                         placeholder="no-reply@mydomain.com"
-                        onChange={(event) => updateSettings('notification_email_from', event.target.value)}
-                     />
+                        onChange={(value:string) => updateSettings('notification_email_from', value)}
+                        />
                   </div>
                </>
             )}

components/settings/ScraperSettings.tsx

@@ -2,6 +2,8 @@ import React from 'react';
 import { useClearFailedQueue } from '../../services/settings';
 import Icon from '../common/Icon';
 import SelectField, { SelectionOption } from '../common/SelectField';
+import SecretField from '../common/SecretField';
+import ToggleField from '../common/ToggleField';
 
 type ScraperSettingsProps = {
    settings: SettingsType,
@@ -43,29 +45,25 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
       <div className='settings__content styled-scrollbar p-6 text-sm'>
 
          <div className="settings__section__select mb-5">
-            <label className={labelStyle}>Scraping Method</label>
             <SelectField
+            label='Scraping Method'
             options={scraperOptions}
             selected={[settings.scraper_type || 'none']}
             defaultLabel="Select Scraper"
             updateField={(updatedTime:[string]) => updateSettings('scraper_type', updatedTime[0])}
             multiple={false}
             rounded={'rounded'}
-            minWidth={270}
+            minWidth={220}
             />
          </div>
-         {['scrapingant', 'scrapingrobot', 'serply', 'serpapi', 'spaceSerp', 'searchapi'].includes(settings.scraper_type) && (
-            <div className="settings__section__input mr-3">
-               <label className={labelStyle}>Scraper API Key or Token</label>
-               <input
-                  className={`w-full p-2 border border-gray-200 rounded mt-2 mb-3 focus:outline-none  focus:border-blue-200 
-                  ${settingsError?.type === 'no_api_key' ? ' border-red-400 focus:border-red-400' : ''} `}
-                  type="text"
-                  value={settings?.scaping_api || ''}
-                  placeholder={'API Key/Token'}
-                  onChange={(event) => updateSettings('scaping_api', event.target.value)}
-               />
-            </div>
+         {settings.scraper_type !== 'none' && settings.scraper_type !== 'proxy' && (
+            <SecretField
+            label='Scraper API Key or Token'
+            placeholder={'API Key/Token'}
+            value={settings?.scaping_api || ''}
+            hasError={settingsError?.type === 'no_api_key'}
+            onChange={(value:string) => updateSettings('scaping_api', value)}
+            />
          )}
          {settings.scraper_type === 'proxy' && (
             <div className="settings__section__input mb-5">
@@ -82,8 +80,8 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
          )}
          {settings.scraper_type !== 'none' && (
             <div className="settings__section__input mb-5">
-               <label className={labelStyle}>Scraping Frequency</label>
                <SelectField
+                  label='Scraping Frequency'
                   multiple={false}
                   selected={[settings?.scrape_interval || 'daily']}
                   options={scrapingOptions}
@@ -91,14 +89,14 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
                   updateField={(updated:string[]) => updated[0] && updateSettings('scrape_interval', updated[0])}
                   rounded='rounded'
                   maxHeight={48}
-                  minWidth={270}
+                  minWidth={220}
                />
                <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
             </div>
          )}
             <div className="settings__section__input mb-5">
-               <label className={labelStyle}>Delay Between Each keyword Scrape</label>
                <SelectField
+                  label='keyword Scrape Delay'
                   multiple={false}
                   selected={[settings?.scrape_delay || '0']}
                   options={delayOptions}
@@ -106,28 +104,16 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
                   updateField={(updated:string[]) => updated[0] && updateSettings('scrape_delay', updated[0])}
                   rounded='rounded'
                   maxHeight={48}
-                  minWidth={270}
+                  minWidth={220}
                />
                <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
             </div>
             <div className="settings__section__input mb-5">
-               <label className="relative inline-flex items-center cursor-pointer w-full justify-between">
-                  <span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-56">Auto Retry Failed Keyword Scrape</span>
-                  <input
-                  type="checkbox"
-                  value={settings?.scrape_retry ? 'true' : '' }
-                  checked={settings.scrape_retry || false}
-                  className="sr-only peer"
-                  onChange={() => updateSettings('scrape_retry', !settings.scrape_retry)}
-                  />
-                  <div className="relative rounded-3xl w-9 h-5 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
-                  peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
-                  peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
-                  after:absolute after:top-[2px] after:left-[2px] after:bg-white  after:border-gray-300
-                  after:border after:rounded-full after:h-4 after:w-4
-                  after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
-
-               </label>
+               <ToggleField
+               label='Auto Retry Failed Keyword Scrape'
+               value={settings?.scrape_retry ? 'true' : '' }
+               onChange={(val) => updateSettings('scrape_retry', val)}
+               />
             </div>
             {settings?.scrape_retry && (settings.failed_queue?.length || 0) > 0 && (
                <div className="settings__section__input mb-5">

components/settings/SearchConsoleSettings.tsx

@@ -0,0 +1,49 @@
+import React from 'react';
+import ToggleField from '../common/ToggleField';
+import InputField from '../common/InputField';
+
+type SearchConsoleSettingsProps = {
+   settings: SettingsType,
+   settingsError: null | {
+      type: string,
+      msg: string
+   },
+   updateSettings: Function,
+}
+
+const SearchConsoleSettings = ({ settings, settingsError, updateSettings }:SearchConsoleSettingsProps) => {
+   return (
+      <div>
+      <div className='settings__content styled-scrollbar p-6 text-sm'>
+
+         {/* <div className="settings__section__input mb-5">
+            <ToggleField
+            label='Enable Goolge Search Console'
+            value={settings?.scrape_retry ? 'true' : '' }
+            onChange={(val) => updateSettings('scrape_retry', val)}
+            />
+         </div> */}
+         <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+            <InputField
+            label='Search Console Client Email'
+            onChange={(client_email:string) => updateSettings('search_console_client_email', client_email)}
+            value={settings.search_console_client_email}
+            placeholder='myapp@appspot.gserviceaccount.com'
+            />
+         </div>
+         <div className="settings__section__input mb-4 flex flex-col justify-between items-center w-full">
+            <label className='mb-2 font-semibold block text-sm text-gray-700 capitalize w-full'>Search Console Private Key</label>
+            <textarea
+               className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
+               focus:outline-none h-[100px] focus:border-blue-200`}
+               value={settings.search_console_private_key}
+               placeholder={'-----BEGIN PRIVATE KEY-----/ssssaswdkihad....'}
+               onChange={(event) => updateSettings('search_console_private_key', event.target.value)}
+            />
+         </div>
+      </div>
+   </div>
+   );
+};
+
+export default SearchConsoleSettings;

components/settings/Settings.tsx

@@ -1,9 +1,11 @@
 import React, { useEffect, useState } from 'react';
 import { Toaster } from 'react-hot-toast';
-import useUpdateSettings, { useFetchSettings } from '../../services/settings';
+import { useFetchSettings, useUpdateSettings } from '../../services/settings';
 import Icon from '../common/Icon';
 import NotificationSettings from './NotificationSettings';
 import ScraperSettings from './ScraperSettings';
+import useOnKey from '../../hooks/useOnKey';
+import SearchConsoleSettings from './SearchConsoleSettings';
 
 type SettingsProps = {
    closeSettings: Function,
@@ -15,7 +17,7 @@ type SettingsError = {
    msg: string
 }
 
-const defaultSettings = {
+const defaultSettings: SettingsType = {
    scraper_type: 'none',
    scrape_delay: 'none',
    scrape_retry: false,
@@ -26,6 +28,9 @@ const defaultSettings = {
    smtp_username: '',
    smtp_password: '',
    notification_email_from: '',
+   search_console: true,
+   search_console_client_email: '',
+   search_console_private_key: '',
 };
 
 const Settings = ({ closeSettings }:SettingsProps) => {
@@ -34,6 +39,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
    const [settingsError, setSettingsError] = useState<SettingsError|null>(null);
    const { mutate: updateMutate, isLoading: isUpdating } = useUpdateSettings(() => console.log(''));
    const { data: appSettings, isLoading } = useFetchSettings();
+   useOnKey('Escape', closeSettings);
 
    useEffect(() => {
       if (appSettings && appSettings.settings) {
@@ -41,19 +47,6 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       }
    }, [appSettings]);
 
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            console.log(event.key);
-            closeSettings();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeSettings]);
-
    const closeOnBGClick = (e:React.SyntheticEvent) => {
       e.stopPropagation();
       e.nativeEvent.stopImmediatePropagation();
@@ -92,30 +85,38 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       }
    };
 
-   const tabStyle = 'inline-block px-4 py-1 rounded-full mr-3 cursor-pointer text-sm';
+   const tabStyle = `inline-block px-3 py-2 rounded-md  cursor-pointer text-xs lg:text-sm lg:mr-3 lg:px-4 select-none z-10
+   text-gray-600 border border-b-0 relative top-[1px] rounded-b-none`;
+   const tabStyleActive = 'bg-white text-blue-600 border-slate-200';
+
    return (
        <div className="settings fixed w-full h-screen top-0 left-0 z-50" onClick={closeOnBGClick}>
-            <div className="absolute w-full max-w-xs bg-white customShadow top-0 right-0 h-screen" data-loading={isLoading} >
+            <div className="absolute w-full max-w-md bg-white customShadow top-0 right-0 h-screen" data-loading={isLoading} >
                {isLoading && <div className='absolute flex content-center items-center h-full'><Icon type="loading" size={24} /></div>}
-               <div className='settings__header p-6 border-b border-b-slate-200 text-slate-500'>
+               <div className='settings__header px-5 py-4 text-slate-500'>
                   <h3 className=' text-black text-lg font-bold'>Settings</h3>
                   <button
-                  className=' absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
+                  className=' absolute top-2 right-2 p-2 px- text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
                   onClick={() => closeSettings()}>
                      <Icon type='close' size={24} />
                   </button>
                </div>
-               <div className=' px-4 mt-4 '>
+               <div className='border border-slate-200 px-3 py-4 pb-0 border-l-0 border-r-0 bg-[#f8f9ff]'>
                   <ul>
                      <li
-                     className={`${tabStyle} ${currentTab === 'scraper' ? ' bg-blue-50 text-blue-600' : ''}`}
+                     className={`${tabStyle} ${currentTab === 'scraper' ? tabStyleActive : 'border-transparent '}`}
                      onClick={() => setCurrentTab('scraper')}>
-                        Scraper
+                       <Icon type='scraper' /> Scraper
                      </li>
                      <li
-                     className={`${tabStyle} ${currentTab === 'notification' ? ' bg-blue-50 text-blue-600' : ''}`}
+                     className={`${tabStyle} ${currentTab === 'notification' ? tabStyleActive : 'border-transparent'}`}
                      onClick={() => setCurrentTab('notification')}>
-                        Notification
+                        <Icon type='email' /> Notification
+                     </li>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'searchconsole' ? tabStyleActive : 'border-transparent'}`}
+                     onClick={() => setCurrentTab('searchconsole')}>
+                       <Icon type='google' size={14} /> Search Console
                      </li>
                   </ul>
                </div>
@@ -126,6 +127,9 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                {currentTab === 'notification' && settings && (
                   <NotificationSettings settings={settings} updateSettings={updateSettings} settingsError={settingsError} />
                )}
+               {currentTab === 'searchconsole' && settings && (
+                  <SearchConsoleSettings settings={settings} updateSettings={updateSettings} settingsError={settingsError} />
+               )}
                <div className=' border-t-[1px] border-gray-200 p-2 px-3'>
                   <button
                   onClick={() => performUpdate()}

database/config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  production: {
+    username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER,
+    password: process.env.PASSWORD,
+    database: 'sequelize',
+    host: 'database',
+    port: 3306,
+    dialect: 'sqlite',
+    storage: './data/database.sqlite',
+    dialectOptions: {
+      bigNumberStrings: true,
+    },
+  },
+};

database/migrations/1707068556345-add-new-keyword-fields.js

@@ -0,0 +1,45 @@
+// Migration: Adds city, latlong and settings keyword to keyword table.
+
+// CLI Migration
+module.exports = {
+   up: async (queryInterface, Sequelize) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (!keywordTableDefinition.city) {
+                  await queryInterface.addColumn('keyword', 'city', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+               if (!keywordTableDefinition.latlong) {
+                  await queryInterface.addColumn('keyword', 'latlong', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+               if (!keywordTableDefinition.settings) {
+                  await queryInterface.addColumn('keyword', 'settings', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+   down: (queryInterface) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (keywordTableDefinition.city) {
+                  await queryInterface.removeColumn('keyword', 'city', { transaction: t });
+               }
+               if (keywordTableDefinition.latlong) {
+                  await queryInterface.removeColumn('keyword', 'latlong', { transaction: t });
+               }
+               if (keywordTableDefinition.latlong) {
+                  await queryInterface.removeColumn('keyword', 'settings', { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+ };

database/migrations/1707233039698-add-domain-searchconsole-field.js

@@ -0,0 +1,29 @@
+// Migration: Adds search_console field to domain table to assign search console property type, url and api.
+
+// CLI Migration
+module.exports = {
+   up: (queryInterface, Sequelize) => {
+     return queryInterface.sequelize.transaction(async (t) => {
+      try {
+         const domainTableDefinition = await queryInterface.describeTable('domain');
+         if (domainTableDefinition && !domainTableDefinition.search_console) {
+            await queryInterface.addColumn('domain', 'search_console', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+         }
+      } catch (error) {
+         console.log('error :', error);
+      }
+     });
+   },
+   down: (queryInterface) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const domainTableDefinition = await queryInterface.describeTable('domain');
+            if (domainTableDefinition && domainTableDefinition.search_console) {
+               await queryInterface.removeColumn('domain', 'search_console', { transaction: t });
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+ };

database/models/domain.ts

@@ -38,6 +38,9 @@ class Domain extends Model {
 
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
    notification_emails!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true })
+   search_console!: string;
 }
 
 export default Domain;

database/models/keyword.ts

@@ -19,7 +19,13 @@ class Keyword extends Model {
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: 'US' })
    country!: string;
 
-   @Column({ type: DataType.STRING, allowNull: false })
+   @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
+   city!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
+   latlong!: string;
+
+   @Column({ type: DataType.STRING, allowNull: false, defaultValue: '{}' })
    domain!: string;
 
    // @ForeignKey(() => Domain)
@@ -58,6 +64,9 @@ class Keyword extends Model {
 
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: 'false' })
    lastUpdateError!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true })
+   settings!: string;
 }
 
 export default Keyword;

entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+npx sequelize-cli db:migrate --env production
+exec "$@"

hooks/useIsMobile.tsx

@@ -0,0 +1,12 @@
+import { useEffect, useState } from 'react';
+
+const useIsMobile = () => {
+   const [isMobile, setIsMobile] = useState<boolean>(false);
+   useEffect(() => {
+      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
+   }, []);
+
+   return [isMobile];
+};
+
+export default useIsMobile;

hooks/useOnKey.tsx

@@ -0,0 +1,17 @@
+import { useEffect } from 'react';
+
+const useOnKey = (key:string, onPress: Function) => {
+   useEffect(() => {
+      const closeModalonEsc = (event:KeyboardEvent) => {
+         if (event.key === key) {
+            onPress();
+         }
+      };
+      window.addEventListener('keydown', closeModalonEsc, false);
+      return () => {
+         window.removeEventListener('keydown', closeModalonEsc, false);
+      };
+   }, [key, onPress]);
+};
+
+export default useOnKey;

hooks/useWindowResize.tsx

@@ -0,0 +1,13 @@
+import { useEffect } from 'react';
+
+const useWindowResize = (onResize: () => void) => {
+   useEffect(() => {
+      onResize();
+      window.addEventListener('resize', onResize);
+      return () => {
+          window.removeEventListener('resize', onResize);
+      };
+   }, [onResize]);
+};
+
+export default useWindowResize;

jest.setup.js

@@ -1,11 +1,26 @@
 // eslint-disable-next-line no-unused-vars
 import 'isomorphic-fetch';
 import './styles/globals.css';
+import '@testing-library/jest-dom';
+import { enableFetchMocks } from 'jest-fetch-mock';
 // Optional: configure or set up a testing framework before each test.
 // If you delete this file, remove `setupFilesAfterEnv` from `jest.config.js`
 
 // Used for __tests__/testing-library.js
 // Learn more: https://github.com/testing-library/jest-dom
-import '@testing-library/jest-dom';
+
+window.matchMedia = (query) => ({
+   matches: false,
+   media: query,
+   onchange: null,
+   addListener: jest.fn(), // deprecated
+   removeListener: jest.fn(), // deprecated
+   addEventListener: jest.fn(),
+   removeEventListener: jest.fn(),
+   dispatchEvent: jest.fn(),
+});
 
 global.ResizeObserver = require('resize-observer-polyfill');
+
+// Enable Fetch Mocking
+enableFetchMocks();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "0.3.2",
+  "version": "1.0.2",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -13,6 +13,8 @@
     "test": "jest --watch --verbose",
     "test:ci": "jest --ci",
     "test:cv": "jest --coverage --coverageDirectory='coverage'",
+    "db:migrate": "sequelize-cli db:migrate --env production",
+    "db:revert": "sequelize-cli db:migrate:undo --env production",
     "release": "standard-version"
   },
   "dependencies": {
@@ -33,7 +35,7 @@
     "jsonwebtoken": "^9.0.2",
     "msw": "^0.49.0",
     "next": "^12.3.4",
-    "nodemailer": "^6.8.0",
+    "nodemailer": "^6.9.9",
     "react": "18.2.0",
     "react-chartjs-2": "^4.3.1",
     "react-dom": "18.2.0",
@@ -44,8 +46,9 @@
     "react-window": "^1.8.8",
     "reflect-metadata": "^0.1.13",
     "sequelize": "^6.34.0",
-    "sequelize-typescript": "^2.1.5",
-    "sqlite3": "^5.1.6"
+    "sequelize-typescript": "^2.1.6",
+    "sqlite3": "^5.1.6",
+    "umzug": "^3.6.1"
   },
   "devDependencies": {
     "@testing-library/jest-dom": "^6.1.4",
@@ -67,10 +70,13 @@
     "eslint-config-next": "12.3.1",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
+    "jest-fetch-mock": "^3.0.3",
+    "next-router-mock": "^0.9.10",
     "postcss": "^8.4.31",
     "prettier": "^2.7.1",
     "resize-observer-polyfill": "^1.5.1",
     "sass": "^1.55.0",
+    "sequelize-cli": "^6.6.2",
     "standard-version": "^9.5.0",
     "stylelint-config-standard": "^29.0.0",
     "tailwindcss": "^3.1.8",

pages/api/dbmigrate.ts

@@ -0,0 +1,53 @@
+import { Sequelize } from 'sequelize';
+import { Umzug, SequelizeStorage } from 'umzug';
+import type { NextApiRequest, NextApiResponse } from 'next';
+import db from '../../database/database';
+import verifyUser from '../../utils/verifyUser';
+
+type MigrationGetResponse = {
+   hasMigrations: boolean,
+}
+
+type MigrationPostResponse = {
+   migrated: boolean,
+   erroor?: string
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized === 'authorized' && req.method === 'GET') {
+      await db.sync();
+      return getMigrationStatus(req, res);
+   }
+   if (authorized === 'authorized' && req.method === 'POST') {
+      return migrateDatabase(req, res);
+   }
+   return res.status(401).json({ error: authorized });
+}
+
+const getMigrationStatus = async (req: NextApiRequest, res: NextApiResponse<MigrationGetResponse>) => {
+   const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false });
+   const umzug = new Umzug({
+      migrations: { glob: 'database/migrations/*.js' },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: undefined,
+   });
+   const migrations = await umzug.pending();
+   // console.log('migrations :', migrations);
+   // const migrationsExceuted = await umzug.executed();
+   return res.status(200).json({ hasMigrations: migrations.length > 0 });
+};
+
+const migrateDatabase = async (req: NextApiRequest, res: NextApiResponse<MigrationPostResponse>) => {
+   const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false });
+   const umzug = new Umzug({
+      migrations: { glob: 'database/migrations/*.js' },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: undefined,
+   });
+   const migrations = await umzug.up();
+   console.log('[Updated] migrations :', migrations);
+   return res.status(200).json({ migrated: true });
+};

pages/api/domain.ts

@@ -0,0 +1,48 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import Cryptr from 'cryptr';
+import db from '../../database/database';
+import Domain from '../../database/models/domain';
+import verifyUser from '../../utils/verifyUser';
+
+type DomainGetResponse = {
+   domain?: DomainType | null
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized === 'authorized' && req.method === 'GET') {
+      await db.sync();
+      return getDomain(req, res);
+   }
+   return res.status(401).json({ error: authorized });
+}
+
+const getDomain = async (req: NextApiRequest, res: NextApiResponse<DomainGetResponse>) => {
+   if (!req.query.domain && typeof req.query.domain !== 'string') {
+       return res.status(400).json({ error: 'Domain Name is Required!' });
+   }
+
+   try {
+      const query = { domain: req.query.domain as string };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const parsedDomain = foundDomain?.get({ plain: true }) || false;
+
+      if (parsedDomain && parsedDomain.search_console) {
+         try {
+            const cryptr = new Cryptr(process.env.SECRET as string);
+            const scData = JSON.parse(parsedDomain.search_console);
+            scData.client_email = scData.client_email ? cryptr.decrypt(scData.client_email) : '';
+            scData.private_key = scData.private_key ? cryptr.decrypt(scData.private_key) : '';
+            parsedDomain.search_console = JSON.stringify(scData);
+         } catch (error) {
+            console.log('[Error] Parsing Search Console Keys.');
+         }
+      }
+
+      return res.status(200).json({ domain: parsedDomain });
+   } catch (error) {
+      console.log('[ERROR] Getting Domain: ', error);
+      return res.status(400).json({ error: 'Error Loading Domain' });
+   }
+};

pages/api/domains.ts

@@ -1,9 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
+import Cryptr from 'cryptr';
 import db from '../../database/database';
 import Domain from '../../database/models/domain';
 import Keyword from '../../database/models/keyword';
 import getdomainStats from '../../utils/domains';
 import verifyUser from '../../utils/verifyUser';
+import { checkSerchConsoleIntegration, removeLocalSCData } from '../../utils/searchConsole';
 
 type DomainsGetRes = {
    domains: DomainType[]
@@ -11,13 +13,14 @@ type DomainsGetRes = {
 }
 
 type DomainsAddResponse = {
-   domain: Domain|null,
+   domains: DomainType[]|null,
    error?: string|null,
 }
 
 type DomainsDeleteRes = {
    domainRemoved: number,
    keywordsRemoved: number,
+   SCDataRemoved: boolean,
    error?: string|null,
 }
 
@@ -51,7 +54,13 @@ export const getDomains = async (req: NextApiRequest, res: NextApiResponse<Domai
    const withStats = !!req?.query?.withstats;
    try {
       const allDomains: Domain[] = await Domain.findAll();
-      const formattedDomains: DomainType[] = allDomains.map((el) => el.get({ plain: true }));
+      const formattedDomains: DomainType[] = allDomains.map((el) => {
+         const domainItem:DomainType = el.get({ plain: true });
+         const scData = domainItem?.search_console ? JSON.parse(domainItem.search_console) : {};
+         const { client_email, private_key } = scData;
+         const searchConsoleData = scData ? { ...scData, client_email: client_email ? 'true' : '', private_key: private_key ? 'true' : '' } : {};
+         return { ...domainItem, search_console: JSON.stringify(searchConsoleData) };
+      });
       const theDomains: DomainType[] = withStats ? await getdomainStats(formattedDomains) : allDomains;
       return res.status(200).json({ domains: theDomains });
    } catch (error) {
@@ -59,41 +68,45 @@ export const getDomains = async (req: NextApiRequest, res: NextApiResponse<Domai
    }
 };
 
-export const addDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsAddResponse>) => {
-   if (!req.body.domain) {
-      return res.status(400).json({ domain: null, error: 'Error Adding Domain.' });
-   }
-   const { domain } = req.body || {};
-   const domainData = {
-      domain: domain.trim(),
-      slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-'),
-      lastUpdated: new Date().toJSON(),
-      added: new Date().toJSON(),
-   };
+const addDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsAddResponse>) => {
+   const { domains } = req.body;
+   if (domains && Array.isArray(domains) && domains.length > 0) {
+      const domainsToAdd: any = [];
 
-   try {
-      const addedDomain = await Domain.create(domainData);
-      return res.status(201).json({ domain: addedDomain });
-   } catch (error) {
-      return res.status(400).json({ domain: null, error: 'Error Adding Domain.' });
+      domains.forEach((domain: string) => {
+         domainsToAdd.push({
+            domain: domain.trim(),
+            slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-').replaceAll('/', '-'),
+            lastUpdated: new Date().toJSON(),
+            added: new Date().toJSON(),
+         });
+      });
+      try {
+         const newDomains:Domain[] = await Domain.bulkCreate(domainsToAdd);
+         const formattedDomains = newDomains.map((el) => el.get({ plain: true }));
+         return res.status(201).json({ domains: formattedDomains });
+      } catch (error) {
+         console.log('[ERROR] Adding New Domain ', error);
+         return res.status(400).json({ domains: [], error: 'Error Adding Domain.' });
+      }
+   } else {
+      return res.status(400).json({ domains: [], error: 'Necessary data missing.' });
    }
 };
 
 export const deleteDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsDeleteRes>) => {
    if (!req.query.domain && typeof req.query.domain !== 'string') {
-      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, error: 'Domain is Required!' });
+      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, SCDataRemoved: false, error: 'Domain is Required!' });
    }
    try {
       const { domain } = req.query || {};
       const removedDomCount: number = await Domain.destroy({ where: { domain } });
       const removedKeywordCount: number = await Keyword.destroy({ where: { domain } });
-      return res.status(200).json({
-            domainRemoved: removedDomCount,
-            keywordsRemoved: removedKeywordCount,
-         });
+      const SCDataRemoved = await removeLocalSCData(domain as string);
+      return res.status(200).json({ domainRemoved: removedDomCount, keywordsRemoved: removedKeywordCount, SCDataRemoved });
    } catch (error) {
       console.log('[ERROR] Deleting Domain: ', req.query.domain, error);
-      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, error: 'Error Deleting Domain' });
+      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, SCDataRemoved: false, error: 'Error Deleting Domain' });
    }
 };
 
@@ -102,17 +115,28 @@ export const updateDomain = async (req: NextApiRequest, res: NextApiResponse<Dom
       return res.status(400).json({ domain: null, error: 'Domain is Required!' });
    }
    const { domain } = req.query || {};
-   const { notification_interval, notification_emails } = req.body;
+   const { notification_interval, notification_emails, search_console } = req.body as DomainSettings;
 
    try {
       const domainToUpdate: Domain|null = await Domain.findOne({ where: { domain } });
+      // Validate Search Console API Data
+      if (domainToUpdate && search_console?.client_email && search_console?.private_key) {
+         const theDomainObj = domainToUpdate.get({ plain: true });
+         const isSearchConsoleAPIValid = await checkSerchConsoleIntegration({ ...theDomainObj, search_console: JSON.stringify(search_console) });
+         if (!isSearchConsoleAPIValid.isValid) {
+            return res.status(400).json({ domain: null, error: isSearchConsoleAPIValid.error });
+         }
+         const cryptr = new Cryptr(process.env.SECRET as string);
+         search_console.client_email = search_console.client_email ? cryptr.encrypt(search_console.client_email.trim()) : '';
+         search_console.private_key = search_console.private_key ? cryptr.encrypt(search_console.private_key.trim()) : '';
+      }
       if (domainToUpdate) {
-         domainToUpdate.set({ notification_interval, notification_emails });
+         domainToUpdate.set({ notification_interval, notification_emails, search_console: JSON.stringify(search_console) });
          await domainToUpdate.save();
       }
       return res.status(200).json({ domain: domainToUpdate });
    } catch (error) {
       console.log('[ERROR] Updating Domain: ', req.query.domain, error);
-      return res.status(400).json({ domain: null, error: 'Error Updating Domain' });
+      return res.status(400).json({ domain: null, error: 'Error Updating Domain. An Unknown Error Occured.' });
    }
 };

pages/api/insight.ts

@@ -1,8 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import db from '../../database/database';
 import { getCountryInsight, getKeywordsInsight, getPagesInsight } from '../../utils/insight';
-import { fetchDomainSCData, readLocalSCData } from '../../utils/searchConsole';
+import { fetchDomainSCData, getSearchConsoleApiInfo, readLocalSCData } from '../../utils/searchConsole';
 import verifyUser from '../../utils/verifyUser';
+import Domain from '../../database/models/domain';
 
 type SCInsightRes = {
    data: InsightDataType | null,
@@ -23,9 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
 const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiResponse<SCInsightRes>) => {
    if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' });
-   if (!!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) === false) {
-      return res.status(200).json({ data: null, error: 'Google Search Console Not Integrated' });
-   }
    const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
    const getInsightFromSCData = (localSCData: SCDomainDataType): InsightDataType => {
       const { stats = [] } = localSCData;
@@ -37,17 +35,26 @@ const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiRe
 
    // First try and read the  Local SC Domain Data file.
    const localSCData = await readLocalSCData(domainname);
-   const oldFetchedDate = localSCData.lastFetched;
-   const fetchTimeDiff = new Date().getTime() - (oldFetchedDate ? new Date(oldFetchedDate as string).getTime() : 0);
 
-   if (localSCData && localSCData.stats && localSCData.stats.length && fetchTimeDiff <= 86400000) {
-      const response = getInsightFromSCData(localSCData);
-      return res.status(200).json({ data: response });
+   if (localSCData) {
+      const oldFetchedDate = localSCData.lastFetched;
+      const fetchTimeDiff = new Date().getTime() - (oldFetchedDate ? new Date(oldFetchedDate as string).getTime() : 0);
+      if (localSCData.stats && localSCData.stats.length && fetchTimeDiff <= 86400000) {
+         const response = getInsightFromSCData(localSCData);
+         return res.status(200).json({ data: response });
+      }
    }
 
    // If the Local SC Domain Data file does not exist, fetch from Googel Search Console.
    try {
-      const scData = await fetchDomainSCData(domainname);
+      const query = { domain: domainname };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
+      const scDomainAPI = await getSearchConsoleApiInfo(domainObj);
+      if (!(scDomainAPI.client_email && scDomainAPI.private_key)) {
+         return res.status(200).json({ data: null, error: 'Google Search Console is not Integrated.' });
+      }
+      const scData = await fetchDomainSCData(domainObj, scDomainAPI);
       const response = getInsightFromSCData(scData);
       return res.status(200).json({ data: response });
    } catch (error) {

pages/api/keywords.ts

@@ -45,7 +45,7 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
    if (!req.query.domain && typeof req.query.domain !== 'string') {
       return res.status(400).json({ error: 'Domain is Required!' });
    }
-   const domain = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
+   const domain = (req.query.domain as string);
    const integratedSC = process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL;
    const domainSCData = integratedSC ? await readLocalSCData(domain) : false;
 
@@ -79,13 +79,14 @@ const addKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
       const keywordsToAdd: any = []; // QuickFIX for bug: https://github.com/sequelize/sequelize-typescript/issues/936
 
       keywords.forEach((kwrd: KeywordAddPayload) => {
-         const { keyword, device, country, domain, tags } = kwrd;
+         const { keyword, device, country, domain, tags, city } = kwrd;
          const tagsArray = tags ? tags.split(',').map((item:string) => item.trim()) : [];
          const newKeyword = {
             keyword,
             device,
             domain,
             country,
+            city,
             position: 0,
             updating: true,
             history: JSON.stringify({}),

pages/api/searchconsole.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import db from '../../database/database';
 import Domain from '../../database/models/domain';
-import { fetchDomainSCData, readLocalSCData } from '../../utils/searchConsole';
+import { fetchDomainSCData, getSearchConsoleApiInfo, readLocalSCData } from '../../utils/searchConsole';
 import verifyUser from '../../utils/verifyUser';
 
 type searchConsoleRes = {
@@ -31,18 +31,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
 const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse<searchConsoleRes>) => {
    if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' });
-   if (!!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) === false) {
-      return res.status(200).json({ data: null, error: 'Google Search Console Not Integrated' });
-   }
    const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
    const localSCData = await readLocalSCData(domainname);
-   console.log(localSCData && localSCData.thirtyDays && localSCData.thirtyDays.length);
-
    if (localSCData && localSCData.thirtyDays && localSCData.thirtyDays.length) {
       return res.status(200).json({ data: localSCData });
    }
    try {
-      const scData = await fetchDomainSCData(domainname);
+      const query = { domain: domainname };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
+      const scDomainAPI = await getSearchConsoleApiInfo(domainObj);
+      if (!(scDomainAPI.client_email && scDomainAPI.private_key)) {
+         return res.status(200).json({ data: null, error: 'Google Search Console is not Integrated.' });
+      }
+      const scData = await fetchDomainSCData(domainObj, scDomainAPI);
       return res.status(200).json({ data: scData });
    } catch (error) {
       console.log('[ERROR] Getting Search Console Data for: ', domainname, error);
@@ -53,9 +55,9 @@ const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiRespo
 const cronRefreshSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse<searchConsoleCRONRes>) => {
    try {
       const allDomainsRaw = await Domain.findAll();
-      const Domains: Domain[] = allDomainsRaw.map((el) => el.get({ plain: true }));
+      const Domains: DomainType[] = allDomainsRaw.map((el) => el.get({ plain: true }));
       for (const domain of Domains) {
-         await fetchDomainSCData(domain.domain);
+         await fetchDomainSCData(domain);
       }
       return res.status(200).json({ status: 'completed' });
    } catch (error) {

pages/api/settings.ts

@@ -42,9 +42,11 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
    }
    try {
       const cryptr = new Cryptr(process.env.SECRET as string);
-      const scaping_api = settings.scaping_api ? cryptr.encrypt(settings.scaping_api) : '';
-      const smtp_password = settings.smtp_password ? cryptr.encrypt(settings.smtp_password) : '';
-      const securedSettings = { ...settings, scaping_api, smtp_password };
+      const scaping_api = settings.scaping_api ? cryptr.encrypt(settings.scaping_api.trim()) : '';
+      const smtp_password = settings.smtp_password ? cryptr.encrypt(settings.smtp_password.trim()) : '';
+      const search_console_client_email = settings.search_console_client_email ? cryptr.encrypt(settings.search_console_client_email.trim()) : '';
+      const search_console_private_key = settings.search_console_private_key ? cryptr.encrypt(settings.search_console_private_key.trim()) : '';
+      const securedSettings = { ...settings, scaping_api, smtp_password, search_console_client_email, search_console_private_key };
 
       await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify(securedSettings), { encoding: 'utf-8' });
       return res.status(200).json({ settings });
@@ -55,6 +57,7 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
 };
 
 export const getAppSettings = async () : Promise<SettingsType> => {
+   const screenshotAPIKey = process.env.SCREENSHOT_API || '69408-serpbear';
    try {
       const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
       const failedQueueRaw = await readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' });
@@ -66,13 +69,19 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          const cryptr = new Cryptr(process.env.SECRET as string);
          const scaping_api = settings.scaping_api ? cryptr.decrypt(settings.scaping_api) : '';
          const smtp_password = settings.smtp_password ? cryptr.decrypt(settings.smtp_password) : '';
+         const search_console_client_email = settings.search_console_client_email ? cryptr.decrypt(settings.search_console_client_email) : '';
+         const search_console_private_key = settings.search_console_private_key ? cryptr.decrypt(settings.search_console_private_key) : '';
          decryptedSettings = {
             ...settings,
             scaping_api,
             smtp_password,
-            search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL),
-            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
+            search_console_client_email,
+            search_console_private_key,
+            search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL)
+            || !!(search_console_client_email && search_console_private_key),
+            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id, allowsCity: !!scraper.allowsCity })),
             failed_queue: failedQueue,
+            screenshot_key: screenshotAPIKey,
          };
       } catch (error) {
          console.log('Error Decrypting Settings API Keys!');
@@ -81,7 +90,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
       return decryptedSettings;
    } catch (error) {
       console.log('[ERROR] Getting App Settings. ', error);
-      const settings = {
+      const settings: SettingsType = {
          scraper_type: 'none',
          notification_interval: 'never',
          notification_email: '',
@@ -91,6 +100,10 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          smtp_username: '',
          smtp_password: '',
          scrape_retry: false,
+         screenshot_key: screenshotAPIKey,
+         search_console: true,
+         search_console_client_email: '',
+         search_console_private_key: '',
       };
       const otherSettings = {
          available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),

pages/domain/[slug]/index.tsx

@@ -1,9 +1,7 @@
-import React, { useEffect, useMemo, useState } from 'react';
+import React, { useMemo, useState } from 'react';
 import type { NextPage } from 'next';
 import Head from 'next/head';
 import { useRouter } from 'next/router';
-// import { useQuery } from 'react-query';
-// import toast from 'react-hot-toast';
 import { CSSTransition } from 'react-transition-group';
 import Sidebar from '../../../components/common/Sidebar';
 import TopBar from '../../../components/common/TopBar';
@@ -11,47 +9,46 @@ import DomainHeader from '../../../components/domains/DomainHeader';
 import KeywordsTable from '../../../components/keywords/KeywordsTable';
 import AddDomain from '../../../components/domains/AddDomain';
 import DomainSettings from '../../../components/domains/DomainSettings';
-import exportCSV from '../../../utils/exportcsv';
+import exportCSV from '../../../utils/client/exportcsv';
 import Settings from '../../../components/settings/Settings';
 import { useFetchDomains } from '../../../services/domains';
 import { useFetchKeywords } from '../../../services/keywords';
 import { useFetchSettings } from '../../../services/settings';
+import AddKeywords from '../../../components/keywords/AddKeywords';
 
 const SingleDomain: NextPage = () => {
    const router = useRouter();
-   const [noScrapprtError, setNoScrapprtError] = useState(false);
    const [showAddKeywords, setShowAddKeywords] = useState(false);
    const [showAddDomain, setShowAddDomain] = useState(false);
    const [showDomainSettings, setShowDomainSettings] = useState(false);
    const [showSettings, setShowSettings] = useState(false);
    const [keywordSPollInterval, setKeywordSPollInterval] = useState<undefined|number>(undefined);
-   const { data: appSettings } = useFetchSettings();
+   const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings();
    const { data: domainsData } = useFetchDomains(router);
-   const { keywordsData, keywordsLoading } = useFetchKeywords(router, setKeywordSPollInterval, keywordSPollInterval);
-
-   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
-   const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
+   const appSettings: SettingsType = appSettingsData?.settings || {};
+   const { scraper_type = '', available_scapers = [] } = appSettings;
+   const activeScraper = useMemo(() => available_scapers.find((scraper) => scraper.value === scraper_type), [scraper_type, available_scapers]);
 
    const activDomain: DomainType|null = useMemo(() => {
       let active:DomainType|null = null;
       if (domainsData?.domains && router.query?.slug) {
-         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug);
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
       }
       return active;
    }, [router.query.slug, domainsData]);
 
-   useEffect(() => {
-      // console.log('appSettings.settings: ', appSettings && appSettings.settings);
-      if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) {
-         setNoScrapprtError(true);
-      }
-   }, [appSettings]);
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
 
-   // console.log('Domains Data:', router, activDomain, theKeywords);
+   const { keywordsData, keywordsLoading } = useFetchKeywords(router, activDomain?.domain || '', setKeywordSPollInterval, keywordSPollInterval);
+   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
+   const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
 
    return (
       <div className="Domain ">
-         {noScrapprtError && (
+         {((!scraper_type || (scraper_type === 'none')) && !isAppSettingsLoading) && (
                <div className=' p-3 bg-red-600 text-white text-sm text-center'>
                   A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
                </div>
@@ -80,13 +77,13 @@ const SingleDomain: NextPage = () => {
                keywords={theKeywords}
                showAddModal={showAddKeywords}
                setShowAddModal={setShowAddKeywords}
-               isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
+               isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) || domainHasScAPI }
                />
             </div>
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
 
          <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
@@ -98,6 +95,15 @@ const SingleDomain: NextPage = () => {
          <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
              <Settings closeSettings={() => setShowSettings(false)} />
          </CSSTransition>
+         <CSSTransition in={showAddKeywords} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddKeywords
+               domain={activDomain?.domain || ''}
+               scraperName={activeScraper?.label || ''}
+               keywords={theKeywords}
+               allowsCity={!!activeScraper?.allowsCity}
+               closeModal={() => setShowAddKeywords(false)}
+               />
+         </CSSTransition>
       </div>
    );
 };

pages/domain/console/[slug]/index.tsx

@@ -10,7 +10,7 @@ import TopBar from '../../../../components/common/TopBar';
 import DomainHeader from '../../../../components/domains/DomainHeader';
 import AddDomain from '../../../../components/domains/AddDomain';
 import DomainSettings from '../../../../components/domains/DomainSettings';
-import exportCSV from '../../../../utils/exportcsv';
+import exportCSV from '../../../../utils/client/exportcsv';
 import Settings from '../../../../components/settings/Settings';
 import { useFetchDomains } from '../../../../services/domains';
 import { useFetchSCKeywords } from '../../../../services/searchConsole';
@@ -34,11 +34,16 @@ const DiscoverPage: NextPage = () => {
    const activDomain: DomainType|null = useMemo(() => {
       let active:DomainType|null = null;
       if (domainsData?.domains && router.query?.slug) {
-         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug);
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
       }
       return active;
    }, [router.query.slug, domainsData]);
 
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
+
    return (
       <div className="Domain ">
          {activDomain && activDomain.domain
@@ -65,13 +70,13 @@ const DiscoverPage: NextPage = () => {
                isLoading={keywordsLoading || isFetching}
                domain={activDomain}
                keywords={theKeywords}
-               isConsoleIntegrated={scConnected}
+               isConsoleIntegrated={scConnected || domainHasScAPI}
                />
             </div>
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
 
          <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>

pages/domain/insight/[slug]/index.tsx

@@ -10,7 +10,7 @@ import TopBar from '../../../../components/common/TopBar';
 import DomainHeader from '../../../../components/domains/DomainHeader';
 import AddDomain from '../../../../components/domains/AddDomain';
 import DomainSettings from '../../../../components/domains/DomainSettings';
-import exportCSV from '../../../../utils/exportcsv';
+import exportCSV from '../../../../utils/client/exportcsv';
 import Settings from '../../../../components/settings/Settings';
 import { useFetchDomains } from '../../../../services/domains';
 import { useFetchSCInsight } from '../../../../services/searchConsole';
@@ -34,11 +34,16 @@ const InsightPage: NextPage = () => {
    const activDomain: DomainType|null = useMemo(() => {
       let active:DomainType|null = null;
       if (domainsData?.domains && router.query?.slug) {
-         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug);
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
       }
       return active;
    }, [router.query.slug, domainsData]);
 
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
+
    return (
       <div className="Domain ">
          {activDomain && activDomain.domain
@@ -65,13 +70,13 @@ const InsightPage: NextPage = () => {
                isLoading={false}
                domain={activDomain}
                insight={theInsight}
-               isConsoleIntegrated={scConnected}
+               isConsoleIntegrated={scConnected || domainHasScAPI}
                />
             </div>
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
 
          <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>

pages/domains/index.tsx

@@ -1,69 +1,92 @@
-import React, { useEffect, useState } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
 import type { NextPage } from 'next';
 import Head from 'next/head';
 import { useRouter } from 'next/router';
 import { CSSTransition } from 'react-transition-group';
+import toast, { Toaster } from 'react-hot-toast';
 import TopBar from '../../components/common/TopBar';
 import AddDomain from '../../components/domains/AddDomain';
 import Settings from '../../components/settings/Settings';
-import { useFetchSettings } from '../../services/settings';
-import { useFetchDomains } from '../../services/domains';
+import { useCheckMigrationStatus, useFetchSettings } from '../../services/settings';
+import { fetchDomainScreenshot, useFetchDomains } from '../../services/domains';
 import DomainItem from '../../components/domains/DomainItem';
 import Icon from '../../components/common/Icon';
 
 type thumbImages = { [domain:string] : string }
 
-const SingleDomain: NextPage = () => {
+const Domains: NextPage = () => {
    const router = useRouter();
-   const [noScrapprtError, setNoScrapprtError] = useState(false);
+   // const [noScrapprtError, setNoScrapprtError] = useState(false);
    const [showSettings, setShowSettings] = useState(false);
    const [showAddDomain, setShowAddDomain] = useState(false);
    const [domainThumbs, setDomainThumbs] = useState<thumbImages>({});
-   const { data: appSettings } = useFetchSettings();
+   const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings();
    const { data: domainsData, isLoading } = useFetchDomains(router, true);
+   const { data: migrationStatus } = useCheckMigrationStatus();
+   // const { mutate: updateDatabaseMutate, isLoading: isUpdatingDB } = useMigrateDatabase((res:Object) => { window.location.reload(); });
+
+   const appSettings:SettingsType = appSettingsData?.settings || {};
+   const { scraper_type = '' } = appSettings;
+
+   const totalKeywords = useMemo(() => {
+      let keywords = 0;
+      if (domainsData?.domains) {
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            keywords += domain?.keywordCount || 0;
+         });
+      }
+      return keywords;
+   }, [domainsData]);
+
+   const domainSCAPiObj = useMemo(() => {
+      const domainsSCAPI:{ [ID:string] : boolean } = {};
+      if (domainsData?.domains) {
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            const doaminSc = domain?.search_console ? JSON.parse(domain.search_console) : {};
+            domainsSCAPI[domain.ID] = doaminSc.client_email && doaminSc.private_key;
+         });
+      }
+      return domainsSCAPI;
+   }, [domainsData]);
 
    useEffect(() => {
-      // console.log('Domains Data: ', domainsData);
-      if (domainsData?.domains && domainsData.domains.length > 0) {
-         const domainThumbsRaw = localStorage.getItem('domainThumbs');
-         const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
+      if (domainsData?.domains && domainsData.domains.length > 0 && appSettings.screenshot_key) {
          domainsData.domains.forEach(async (domain:DomainType) => {
             if (domain.domain) {
-               if (!domThumbs[domain.domain]) {
-                  const domainImageBlob = await fetch(`https://image.thum.io/get/auth/66909-serpbear/maxAge/96/width/200/https://${domain.domain}`).then((res) => res.blob());
-                  if (domainImageBlob) {
-                     const reader = new FileReader();
-                     await new Promise((resolve, reject) => {
-                       reader.onload = resolve;
-                       reader.onerror = reject;
-                       reader.readAsDataURL(domainImageBlob);
-                     });
-                     const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
-                     localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain.domain]: imageBase }));
-                     setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: imageBase }));
-                  }
-               } else {
-                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domThumbs[domain.domain] }));
+               const domainThumb = await fetchDomainScreenshot(domain.domain, appSettings.screenshot_key || '');
+               if (domainThumb) {
+                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domainThumb }));
                }
             }
          });
       }
-   }, [domainsData]);
+   }, [domainsData, appSettings.screenshot_key]);
 
-   useEffect(() => {
-      // console.log('appSettings.settings: ', appSettings && appSettings.settings);
-      if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) {
-         setNoScrapprtError(true);
+   const manuallyUpdateThumb = async (domain: string) => {
+      if (domain && appSettings.screenshot_key) {
+         const domainThumb = await fetchDomainScreenshot(domain, appSettings.screenshot_key, true);
+         if (domainThumb) {
+            toast(`${domain} Screenshot Updated Successfully!`, { icon: '✔️' });
+            setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain]: domainThumb }));
+         } else {
+            toast(`Failed to Fetch ${domain} Screenshot!`, { icon: '⚠️' });
+         }
       }
-   }, [appSettings]);
+   };
 
    return (
-      <div className="Domain flex flex-col min-h-screen">
-         {noScrapprtError && (
+      <div data-testid="domains" className="Domain flex flex-col min-h-screen">
+         {((!scraper_type || (scraper_type === 'none')) && !isAppSettingsLoading) && (
                <div className=' p-3 bg-red-600 text-white text-sm text-center'>
                   A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
                </div>
          )}
+         {migrationStatus?.hasMigrations && (
+               <div className=' p-3 bg-black text-white text-sm text-center'>
+                  You need to Update your database. Stop Serpbear and run this command to update your database:
+                  <code className=' bg-gray-700 px-2 py-0 ml-1'>npm run db:migrate</code>
+               </div>
+         )}
          <Head>
             <title>Domains - SerpBear</title>
          </Head>
@@ -71,9 +94,12 @@ const SingleDomain: NextPage = () => {
 
          <div className="flex flex-col w-full max-w-5xl mx-auto p-6 lg:mt-24 lg:p-0">
             <div className='flex justify-between mb-2 items-center'>
-               <div className=' text-sm'>{domainsData?.domains?.length || 0} Domains</div>
+               <div className=' text-sm text-gray-600'>
+                  {domainsData?.domains?.length || 0} Domains <span className=' text-gray-300 ml-1 mr-1'>|</span> {totalKeywords} keywords
+               </div>
                <div>
                   <button
+                  data-testid="addDomainButton"
                   className={'ml-2 inline-block py-2 text-blue-700 font-bold text-sm'}
                   onClick={() => setShowAddDomain(true)}>
                      <span
@@ -88,8 +114,9 @@ const SingleDomain: NextPage = () => {
                            key={domain.ID}
                            domain={domain}
                            selected={false}
-                           isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
+                           isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) || !!domainSCAPiObj[domain.ID] }
                            thumb={domainThumbs[domain.domain]}
+                           updateThumb={manuallyUpdateThumb}
                            // isConsoleIntegrated={false}
                            />;
                })}
@@ -107,16 +134,17 @@ const SingleDomain: NextPage = () => {
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
          <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
              <Settings closeSettings={() => setShowSettings(false)} />
          </CSSTransition>
          <footer className='text-center flex flex-1 justify-center pb-5 items-end'>
-            <span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings?.settings?.version || '0.0.0'}</a></span>
+            <span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings.version || '0.0.0'}</a></span>
          </footer>
+         <Toaster position='bottom-center' containerClassName="react_toaster" />
       </div>
    );
 };
 
-export default SingleDomain;
+export default Domains;

pages/index.tsx

@@ -8,7 +8,7 @@ import Icon from '../components/common/Icon';
 const Home: NextPage = () => {
    const router = useRouter();
    useEffect(() => {
-      router.push('/domains');
+      if (router) router.push('/domains');
    }, [router]);
 
   return (

scrapers/index.ts

@@ -5,6 +5,8 @@ import serply from './services/serply';
 import spaceserp from './services/spaceserp';
 import proxy from './services/proxy';
 import searchapi from './services/searchapi';
+import valueSerp from './services/valueserp';
+import serper from './services/serper';
 
 export default [
    scrapingRobot,
@@ -14,4 +16,6 @@ export default [
    spaceserp,
    proxy,
    searchapi,
+   valueSerp,
+   serper,
 ];

scrapers/services/searchapi.ts

@@ -1,7 +1,16 @@
+import countries from '../../utils/countries';
+
+interface SearchApiResult {
+   title: string,
+   link: string,
+   position: number,
+ }
+
 const searchapi:ScraperSettings = {
   id: 'searchapi',
   name: 'SearchApi.io',
   website: 'searchapi.io',
+  allowsCity: true,
   headers: (keyword, settings) => {
      return {
         'Content-Type': 'application/json',
@@ -9,7 +18,10 @@ const searchapi:ScraperSettings = {
      };
   },
   scrapeURL: (keyword) => {
-     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}`;
+   const country = keyword.country || 'US';
+   const countryName = countries[country][0];
+   const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
   },
   resultObjectKey: 'organic_results',
   serpExtractor: (content) => {
@@ -29,10 +41,4 @@ const searchapi:ScraperSettings = {
   },
 };
 
-interface SearchApiResult {
-  title: string,
-  link: string,
-  position: number,
-}
-
 export default searchapi;

scrapers/services/serpapi.ts

@@ -1,3 +1,5 @@
+import countries from '../../utils/countries';
+
 interface SerpApiResult {
    title: string,
    link: string,
@@ -8,6 +10,7 @@ const serpapi:ScraperSettings = {
    id: 'serpapi',
    name: 'SerpApi.com',
    website: 'serpapi.com',
+   allowsCity: true,
    headers: (keyword, settings) => {
       return {
          'Content-Type': 'application/json',
@@ -15,7 +18,9 @@ const serpapi:ScraperSettings = {
       };
    },
    scrapeURL: (keyword, settings) => {
-      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}&api_key=${settings.scaping_api}`;
+      const countryName = countries[keyword.country || 'US'][0];
+      const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

scrapers/services/serper.ts

@@ -0,0 +1,35 @@
+interface SerperResult {
+   title: string,
+   link: string,
+   position: number,
+}
+
+const serper:ScraperSettings = {
+   id: 'serper',
+   name: 'Serper.dev',
+   website: 'serper.dev',
+   allowsCity: true,
+   scrapeURL: (keyword, settings, countryData) => {
+      const country = keyword.country || 'US';
+      const lang = countryData[country][2];
+      return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
+   },
+   resultObjectKey: 'organic',
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: SerperResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerperResult[];
+
+      for (const { link, title, position } of results) {
+         if (title && link) {
+            extractedResult.push({
+               title,
+               url: link,
+               position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default serper;

scrapers/services/spaceserp.ts

@@ -1,3 +1,5 @@
+import countries from '../../utils/countries';
+
 interface SpaceSerpResult {
    title: string,
    link: string,
@@ -9,10 +11,14 @@ const spaceSerp:ScraperSettings = {
    id: 'spaceSerp',
    name: 'Space Serp',
    website: 'spaceserp.com',
+   allowsCity: true,
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
+      const countryName = countries[country][0];
+      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const device = keyword.device === 'mobile' ? '&device=mobile' : '';
       const lang = countryData[country][2];
-      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${keyword.device === 'mobile' ? '&device=mobile' : ''}&resultBlocks=`;
+      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

scrapers/services/valueserp.ts

@@ -0,0 +1,41 @@
+import countries from '../../utils/countries';
+
+interface ValueSerpResult {
+   title: string,
+   link: string,
+   position: number,
+   domain: string,
+}
+
+const valueSerp:ScraperSettings = {
+   id: 'valueserp',
+   name: 'Value Serp',
+   website: 'valueserp.com',
+   allowsCity: true,
+   scrapeURL: (keyword, settings, countryData) => {
+      const country = keyword.country || 'US';
+      const countryName = countries[country][0];
+      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const device = keyword.device === 'mobile' ? '&device=mobile' : '';
+      const lang = countryData[country][2];
+      console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
+      return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
+   },
+   resultObjectKey: 'organic_results',
+   serpExtractor: (content) => {
+      const extractedResult = [];
+      const results: ValueSerpResult[] = (typeof content === 'string') ? JSON.parse(content) : content as ValueSerpResult[];
+      for (const result of results) {
+         if (result.title && result.link) {
+            extractedResult.push({
+               title: result.title,
+               url: result.link,
+               position: result.position,
+            });
+         }
+      }
+      return extractedResult;
+   },
+};
+
+export default valueSerp;

services/domains.tsx

@@ -7,7 +7,7 @@ type UpdatePayload = {
    domain: DomainType
 }
 
-export async function fetchDomains(router: NextRouter, withStats:boolean) {
+export async function fetchDomains(router: NextRouter, withStats:boolean): Promise<{domains: DomainType[]}> {
    const res = await fetch(`${window.location.origin}/api/domains${withStats ? '?withstats=true' : ''}`, { method: 'GET' });
    if (res.status >= 400 && res.status < 600) {
       if (res.status === 401) {
@@ -19,16 +19,67 @@ export async function fetchDomains(router: NextRouter, withStats:boolean) {
    return res.json();
 }
 
+export async function fetchDomain(router: NextRouter, domainName: string): Promise<{domain: DomainType}> {
+   if (!domainName) { throw new Error('No Domain Name Provided!'); }
+   const res = await fetch(`${window.location.origin}/api/domain?domain=${domainName}`, { method: 'GET' });
+   if (res.status >= 400 && res.status < 600) {
+      if (res.status === 401) {
+         console.log('Unauthorized!!');
+         router.push('/login');
+      }
+      throw new Error('Bad response from server');
+   }
+   return res.json();
+}
+
+export async function fetchDomainScreenshot(domain: string, screenshot_key:string, forceFetch = false): Promise<string | false> {
+   const domainThumbsRaw = localStorage.getItem('domainThumbs');
+   const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
+   if (!domThumbs[domain] || forceFetch) {
+      try {
+         const screenshotURL = `https://image.thum.io/get/auth/${screenshot_key}/maxAge/96/width/200/https://${domain}`;
+         const domainImageRes = await fetch(screenshotURL);
+         const domainImageBlob = domainImageRes.status === 200 ? await domainImageRes.blob() : false;
+         if (domainImageBlob) {
+            const reader = new FileReader();
+            await new Promise((resolve, reject) => {
+               reader.onload = resolve;
+               reader.onerror = reject;
+               reader.readAsDataURL(domainImageBlob);
+            });
+            const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
+            localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain]: imageBase }));
+            return imageBase;
+         }
+         return false;
+      } catch (error) {
+         return false;
+      }
+   } else if (domThumbs[domain]) {
+         return domThumbs[domain];
+   }
+
+   return false;
+}
+
 export function useFetchDomains(router: NextRouter, withStats:boolean = false) {
    return useQuery('domains', () => fetchDomains(router, withStats));
 }
 
+export function useFetchDomain(router: NextRouter, domainName:string, onSuccess: Function) {
+   return useQuery('domain', () => fetchDomain(router, domainName), {
+      onSuccess: async (data) => {
+         console.log('Domain Loaded!!!', data.domain);
+         onSuccess(data.domain);
+      } });
+}
+
 export function useAddDomain(onSuccess:Function) {
    const router = useRouter();
    const queryClient = useQueryClient();
-   return useMutation(async (domainName:string) => {
+   return useMutation(async (domains:string[]) => {
       const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' });
-      const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ domain: domainName }) };
+      const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ domains }) };
       const res = await fetch(`${window.location.origin}/api/domains`, fetchOpts);
       if (res.status >= 400 && res.status < 600) {
          throw new Error('Bad response from server');
@@ -37,11 +88,12 @@ export function useAddDomain(onSuccess:Function) {
    }, {
       onSuccess: async (data) => {
          console.log('Domain Added!!!', data);
-         const newDomain:DomainType = data.domain;
-         toast(`${newDomain.domain} Added Successfully!`, { icon: '✔️' });
+         const newDomain:DomainType[] = data.domains;
+         const singleDomain = newDomain.length === 1;
+         toast(`${singleDomain ? newDomain[0].domain : `${newDomain.length} domains`} Added Successfully!`, { icon: '✔️' });
          onSuccess(false);
-         if (newDomain && newDomain.slug) {
-            router.push(`/domain/${data.domain.slug}`);
+         if (singleDomain) {
+            router.push(`/domain/${newDomain[0].slug}`);
          }
          queryClient.invalidateQueries(['domains']);
       },
@@ -58,10 +110,11 @@ export function useUpdateDomain(onSuccess:Function) {
       const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' });
       const fetchOpts = { method: 'PUT', headers, body: JSON.stringify(domainSettings) };
       const res = await fetch(`${window.location.origin}/api/domains?domain=${domain.domain}`, fetchOpts);
+      const responseObj = await res.json();
       if (res.status >= 400 && res.status < 600) {
-         throw new Error('Bad response from server');
+         throw new Error(responseObj?.error || 'Bad response from server');
       }
-      return res.json();
+      return responseObj;
    }, {
       onSuccess: async () => {
          console.log('Settings Updated!!!');
@@ -69,8 +122,8 @@ export function useUpdateDomain(onSuccess:Function) {
          onSuccess();
          queryClient.invalidateQueries(['domains']);
       },
-      onError: () => {
-         console.log('Error Updating Domain Settings!!!');
+      onError: (error) => {
+         console.log('Error Updating Domain Settings!!!', error);
          toast('Error Updating Domain Settings', { icon: '⚠️' });
       },
    });

services/keywords.tsx

@@ -2,16 +2,21 @@ import toast from 'react-hot-toast';
 import { NextRouter } from 'next/router';
 import { useMutation, useQuery, useQueryClient } from 'react-query';
 
-export const fetchKeywords = async (router: NextRouter) => {
-   if (!router.query.slug) { return []; }
-   const res = await fetch(`${window.location.origin}/api/keywords?domain=${router.query.slug}`, { method: 'GET' });
+export const fetchKeywords = async (router: NextRouter, domain: string) => {
+   if (!domain) { return []; }
+   const res = await fetch(`${window.location.origin}/api/keywords?domain=${domain}`, { method: 'GET' });
    return res.json();
 };
 
-export function useFetchKeywords(router: NextRouter, setKeywordSPollInterval?:Function, keywordSPollInterval:undefined|number = undefined) {
+export function useFetchKeywords(
+   router: NextRouter,
+   domain: string,
+   setKeywordSPollInterval?:Function,
+   keywordSPollInterval:undefined|number = undefined,
+) {
    const { data: keywordsData, isLoading: keywordsLoading, isError } = useQuery(
-      ['keywords', router.query.slug],
-      () => fetchKeywords(router),
+      ['keywords', domain],
+      () => fetchKeywords(router, domain),
       {
          refetchInterval: keywordSPollInterval,
          onSuccess: (data) => {
@@ -153,3 +158,23 @@ export function useRefreshKeywords(onSuccess:Function) {
       },
    });
 }
+
+export function useFetchSingleKeyword(keywordID:number) {
+   return useQuery(['keyword', keywordID], async () => {
+      try {
+         const fetchURL = `${window.location.origin}/api/keyword?id=${keywordID}`;
+         const res = await fetch(fetchURL, { method: 'GET' }).then((result) => result.json());
+         if (res.status >= 400 && res.status < 600) {
+            throw new Error('Bad response from server');
+         }
+         return { history: res.keyword.history || [], searchResult: res.keyword.lastResult || [] };
+      } catch (error) {
+         throw new Error('Error Loading Keyword Details');
+      }
+   }, {
+      onError: () => {
+         console.log('Error Loading Keyword Data!!!');
+         toast('Error Loading Keyword Details.', { icon: '⚠️' });
+      },
+   });
+}

services/settings.ts

@@ -10,7 +10,7 @@ export function useFetchSettings() {
    return useQuery('settings', () => fetchSettings());
 }
 
-const useUpdateSettings = (onSuccess:Function|undefined) => {
+export const useUpdateSettings = (onSuccess:Function|undefined) => {
    const queryClient = useQueryClient();
 
    return useMutation(async (settings: SettingsType) => {
@@ -61,4 +61,36 @@ export function useClearFailedQueue(onSuccess:Function) {
    });
 }
 
-export default useUpdateSettings;
+export async function fetchMigrationStatus() {
+   const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'GET' });
+   return res.json();
+}
+
+export function useCheckMigrationStatus() {
+   return useQuery('dbmigrate', () => fetchMigrationStatus());
+}
+
+export const useMigrateDatabase = (onSuccess:Function|undefined) => {
+   const queryClient = useQueryClient();
+
+   return useMutation(async () => {
+      // console.log('settings: ', JSON.stringify(settings));
+      const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'POST' });
+      if (res.status >= 400 && res.status < 600) {
+         throw new Error('Bad response from server');
+      }
+      return res.json();
+   }, {
+      onSuccess: async (res) => {
+         if (onSuccess) {
+            onSuccess(res);
+         }
+         toast('Database Updated!', { icon: '✔️' });
+         queryClient.invalidateQueries(['settings']);
+      },
+      onError: () => {
+         console.log('Error Updating Database!!!');
+         toast('Error Updating Database.', { icon: '⚠️' });
+      },
+   });
+};

styles/globals.css

@@ -282,3 +282,8 @@ body {
       right: 240px;
    }
 }
+
+/* Disable LastPass Icon for Secret Field */
+[autocomplete="off"] + div[data-lastpass-icon-root="true"], [autocomplete="off"] + div[data-lastpass-infield="true"] {
+   display: none;
+}

tsconfig.json

@@ -24,7 +24,8 @@
       "next-env.d.ts",
       "**/*.ts",
       "**/*.tsx",
-      "types.d.ts"
+      "types.d.ts",
+      "./jest.setup.js"
    ],
    "exclude": [
       "node_modules"

types.d.ts

@@ -15,6 +15,7 @@ type DomainType = {
    scVisits?: number,
    scImpressions?: number,
    scPosition?: number,
+   search_console?: string,
 }
 
 type KeywordHistory = {
@@ -39,6 +40,7 @@ type KeywordType = {
    lastUpdateError: {date: string, error: string, scraper: string} | false,
    scData?: KeywordSCData,
    uid?: string
+   city?: string
 }
 
 type KeywordLastResult = {
@@ -61,9 +63,17 @@ type countryCodeData = {
    [ISO:string] : string
 }
 
+type DomainSearchConsole = {
+   property_type: 'domain' | 'url',
+   url: string,
+   client_email:string,
+   private_key:string,
+}
+
 type DomainSettings = {
    notification_interval: string,
    notification_emails: string,
+   search_console?: DomainSearchConsole
 }
 
 type SettingsType = {
@@ -77,13 +87,17 @@ type SettingsType = {
    smtp_port: string,
    smtp_username?: string,
    smtp_password?: string,
-   search_console_integrated?: boolean,
-   available_scapers?: Array,
+   available_scapers?: { label: string, value: string, allowsCity?: boolean }[],
    scrape_interval?: string,
    scrape_delay?: string,
    scrape_retry?: boolean,
    failed_queue?: string[]
-   version?: string
+   version?: string,
+   screenshot_key?: string,
+   search_console: boolean,
+   search_console_client_email: string,
+   search_console_private_key: string,
+   search_console_integrated?: boolean,
 }
 
 type KeywordSCDataChild = {
@@ -107,7 +121,8 @@ type KeywordAddPayload = {
    device: string,
    country: string,
    domain: string,
-   tags: string,
+   tags?: string,
+   city?:string
 }
 
 type SearchAnalyticsRawItem = {
@@ -176,11 +191,23 @@ type scraperExtractedItem = {
    position: number,
 }
 interface ScraperSettings {
+   /** A Unique ID for the Scraper. eg: myScraper */
    id:string,
+   /** The Name of the Scraper */
    name:string,
+   /** The Website address of the Scraper */
    website:string,
+   /** The result object's key that contains the results of the scraped data. For example,
+    * if your scraper API the data like this `{scraped:[item1,item2..]}` the resultObjectKey should be "scraped" */
    resultObjectKey: string,
+   /** If the Scraper allows setting a perices location or allows city level scraping set this to true. */
+   allowsCity?: boolean,
+   /** Set your own custom HTTP header properties when making the scraper API request.
+    * The function should return an object that contains all the header properties you want to pass to API request's header.
+    * Example: `{'Cache-Control': 'max-age=0', 'Content-Type': 'application/json'}` */
    headers?(keyword:KeywordType, settings: SettingsType): Object,
+   /** Construct the API URL for scraping the data through your Scraper's API */
    scrapeURL?(keyword:KeywordType, settings:SettingsType, countries:countryData): string,
+   /** Custom function to extract the serp result from the scraped data. The extracted data should be @return {scraperExtractedItem[]} */
    serpExtractor?(content:string): scraperExtractedItem[],
 }

utils/client/exportcsv.ts

@@ -1,4 +1,4 @@
-import countries from './countries';
+import countries from '../countries';
 
   /**
    * Generates CSV File form the given domain & keywords, and automatically downloads it.

utils/client/helpers.ts

@@ -0,0 +1,2 @@
+/* eslint-disable import/prefer-default-export */
+export const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num);

utils/client/validators.ts

@@ -33,3 +33,14 @@ export const isValidDomain = (domain:string): boolean => {
 
    return isValid;
  };
+
+export const isValidUrl = (str: string) => {
+   let url;
+
+   try {
+     url = new URL(str);
+   } catch (e) {
+     return false;
+   }
+   return url.protocol === 'http:' || url.protocol === 'https:';
+ };

utils/domains.ts

@@ -2,6 +2,12 @@ import Keyword from '../database/models/keyword';
 import parseKeywords from './parseKeywords';
 import { readLocalSCData } from './searchConsole';
 
+/**
+ * The function `getdomainStats` takes an array of domain objects, retrieves keyword and stats data for
+ * each domain, and calculates various statistics for each domain.
+ * @param {DomainType[]} domains - An array of objects of type DomainType.
+ * @returns {DomainType[]} - An array of objects of type DomainType.
+ */
 const getdomainStats = async (domains:DomainType[]): Promise<DomainType[]> => {
    const finalDomains: DomainType[] = [];
    console.log('domains: ', domains.length);
@@ -15,7 +21,8 @@ const getdomainStats = async (domains:DomainType[]): Promise<DomainType[]> => {
       domainWithStat.keywordCount = keywords.length;
       const keywordPositions = keywords.reduce((acc, itm) => (acc + itm.position), 0);
       const KeywordsUpdateDates: number[] = keywords.reduce((acc: number[], itm) => [...acc, new Date(itm.lastUpdated).getTime()], [0]);
-      domainWithStat.keywordsUpdated = new Date(Math.max(...KeywordsUpdateDates)).toJSON();
+      const lastKeywordUpdateDate = Math.max(...KeywordsUpdateDates);
+      domainWithStat.keywordsUpdated = new Date(lastKeywordUpdateDate || new Date(domain.lastUpdated).getTime()).toJSON();
       domainWithStat.avgPosition = Math.round(keywordPositions / keywords.length);
 
       // Then Load the SC File and read the stats and calculate the Last 7 days stats

utils/insight.ts

@@ -1,4 +1,10 @@
-export const sortInsightItems = (items:SCInsightItem[], sortBy: string = 'clicks') => {
+/**
+ * The function `sortInsightItems` sorts an array of `SCInsightItem` objects based on a specified property.
+ * @param {SCInsightItem[]} items - An array of SCInsightItem objects.
+ * @param {string} [sortBy=clicks] - The `sortBy` parameter determines the property by which the `items` array should be sorted.
+ * @returns {SCInsightItem[]}
+ */
+export const sortInsightItems = (items:SCInsightItem[], sortBy: string = 'clicks'): SCInsightItem[] => {
    const sortKey = sortBy as keyof SCInsightItem;
    let sortedItems = [];
    switch (sortKey) {
@@ -18,6 +24,13 @@ export const sortInsightItems = (items:SCInsightItem[], sortBy: string = 'clicks
    return sortedItems;
 };
 
+/**
+ * The `getCountryInsight` function takes search analytics data and returns insights about countries based on clicks, impressions, CTR, and position.
+ * @param {SCDomainDataType} SCData - The SCData parameter is an object that contains search analytics data for different dates.
+ * @param {string} [sortBy=clicks] - The "sortBy" parameter is used to specify the sorting criteria for the country insights.
+ * @param {string} [queryDate=thirtyDays] - The `queryDate` parameter is a string that represents the date range for which the search analytics data is retrieved.
+ * @returns {SCInsightItem[]}
+ */
 export const getCountryInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => {
    const keywordsCounts: { [key:string]: string[] } = {};
    const countryItems: { [key:string]: SCInsightItem } = {};
@@ -57,6 +70,13 @@ export const getCountryInsight = (SCData:SCDomainDataType, sortBy:string = 'clic
    return sortBy ? sortInsightItems(countryInsight, sortBy) : countryInsight;
 };
 
+/**
+ * The `getKeywordsInsight` function takes search analytics data, sorts it based on specified criteria, and returns insights on keywords.
+ * @param {SCDomainDataType} SCData - The SCData parameter is of type SCDomainDataType, which is an object containing search analytics data for a specific domain.
+ * @param {string} [sortBy=clicks] - The "sortBy" parameter is used to specify the sorting criteria for the keyword insights.
+ * @param {string} [queryDate=thirtyDays] - The `queryDate` parameter is a string that represents the date range for which the search analytics data is retrieved.
+ * @returns {SCInsightItem[]}
+ */
 export const getKeywordsInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => {
    const keywordItems: { [key:string]: SCInsightItem } = {};
    const keywordCounts: { [key:string]: number } = {};
@@ -99,6 +119,13 @@ export const getKeywordsInsight = (SCData:SCDomainDataType, sortBy:string = 'cli
    return sortBy ? sortInsightItems(keywordInsight, sortBy) : keywordInsight;
 };
 
+/**
+ * The `getPagesInsight` function takes a domain's search analytics data, sorts it based on specified criteria and returns insights about the pages.
+ * @param {SCDomainDataType} SCData - SCData is an object that contains search analytics data for a  specific domain.
+ * @param {string} [sortBy=clicks] - The `sortBy` parameter is used to specify the sorting criteria for the pages insight.
+ * @param {string} [queryDate=thirtyDays] - The `queryDate` parameter is a string that represents the date range for which you want to retrieve the data.
+ * @returns {SCInsightItem[]}
+ */
 export const getPagesInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => {
    const pagesItems: { [key:string]: SCInsightItem } = {};
    const keywordCounts: { [key:string]: number } = {};

utils/scraper.ts

@@ -187,15 +187,21 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
 
 /**
  * Find in the domain's position from the extracted search result.
- * @param {string} domain - Domain Name to look for.
+ * @param {string} domainURL - URL Name to look for.
  * @param {SearchResult[]} result - The search result array extracted from the Google Search result.
  * @returns {SERPObject}
  */
-export const getSerp = (domain:string, result:SearchResult[]) : SERPObject => {
-   if (result.length === 0 || !domain) { return { postion: 0, url: '' }; }
+export const getSerp = (domainURL:string, result:SearchResult[]) : SERPObject => {
+   if (result.length === 0 || !domainURL) { return { postion: 0, url: '' }; }
+   const URLToFind = new URL(domainURL.includes('https://') ? domainURL : `https://${domainURL}`);
+   const theURL = URLToFind.hostname + URLToFind.pathname;
+   const isURL = URLToFind.pathname !== '/';
    const foundItem = result.find((item) => {
-      const itemDomain = item.url.replace('www.', '').match(/^(?:https?:)?(?:\/\/)?([^/?]+)/i);
-      return itemDomain && itemDomain.includes(domain.replace('www.', ''));
+      const itemURL = new URL(item.url.includes('https://') ? item.url : `https://${item.url}`);
+      if (isURL && `${theURL}/` === itemURL.hostname + itemURL.pathname) {
+         return true;
+      }
+      return URLToFind.hostname === itemURL.hostname;
    });
    return { postion: foundItem ? foundItem.position : 0, url: foundItem && foundItem.url ? foundItem.url : '' };
 };

utils/searchConsole.ts

@@ -1,19 +1,39 @@
 import { auth, searchconsole_v1 } from '@googleapis/searchconsole';
-import { readFile, writeFile } from 'fs/promises';
+import Cryptr from 'cryptr';
+import { readFile, writeFile, unlink } from 'fs/promises';
 import { getCountryCodeFromAlphaThree } from './countries';
 
 export type SCDomainFetchError = {
    error: boolean,
    errorMsg: string,
 }
+
+type SCAPISettings = { client_email: string, private_key: string }
+
 type fetchConsoleDataResponse = SearchAnalyticsItem[] | SearchAnalyticsStat[] | SCDomainFetchError;
-const fetchSearchConsoleData = async (domainName:string, days:number, type?:string): Promise<fetchConsoleDataResponse> => {
-   if (!domainName) return { error: true, errorMsg: 'Domain Not Provided!' };
+
+/**
+ * Retrieves data from the Google Search Console API based on the provided domain name, number of days, and optional type.
+ * @param {DomainType} domain - The domain for which you want to fetch search console data.
+ * @param {number} days - number of days of data you want to fetch from the Search Console.
+ * @param {string} [type] - (optional) specifies the type of data to fetch from the Search Console.
+ * @param {SCAPISettings} [api] - (optional) specifies the Seach Console API Information.
+ * @returns {Promise<fetchConsoleDataResponse>}
+ */
+const fetchSearchConsoleData = async (domain:DomainType, days:number, type?:string, api?:SCAPISettings): Promise<fetchConsoleDataResponse> => {
+   if (!domain) return { error: true, errorMsg: 'Domain Not Provided!' };
+   if (!api?.private_key || !api?.client_email) return { error: true, errorMsg: 'Search Console API Data Not Avaialable.' };
+   const domainName = domain.domain;
+   const defaultSCSettings = { property_type: 'domain', url: '', client_email: '', private_key: '' };
+   const domainSettings = domain.search_console ? JSON.parse(domain.search_console) : defaultSCSettings;
+   const sCPrivateKey = api?.private_key || process.env.SEARCH_CONSOLE_PRIVATE_KEY || '';
+   const sCClientEmail = api?.client_email || process.env.SEARCH_CONSOLE_CLIENT_EMAIL || '';
+
    try {
    const authClient = new auth.GoogleAuth({
       credentials: {
-        private_key: process.env.SEARCH_CONSOLE_PRIVATE_KEY ? process.env.SEARCH_CONSOLE_PRIVATE_KEY.replaceAll('\\n', '\n') : '',
-        client_email: process.env.SEARCH_CONSOLE_CLIENT_EMAIL ? process.env.SEARCH_CONSOLE_CLIENT_EMAIL : '',
+        private_key: (sCPrivateKey).replaceAll('\\n', '\n'),
+        client_email: (sCClientEmail || '').trim(),
       },
       scopes: [
         'https://www.googleapis.com/auth/webmasters.readonly',
@@ -42,7 +62,8 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
       };
    }
 
-      const res = client.searchanalytics.query({ siteUrl: `sc-domain:${domainName}`, requestBody });
+      const siteUrl = domainSettings.property_type === 'url' && domainSettings.url ? domainSettings.url : `sc-domain:${domainName}`;
+      const res = client.searchanalytics.query({ siteUrl, requestBody });
       const resData:any = (await res).data;
       let finalRows = resData.rows ? resData.rows.map((item:SearchAnalyticsRawItem) => parseSearchConsoleItem(item, domainName)) : [];
 
@@ -61,19 +82,29 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
       }
 
       return finalRows;
-   } catch (error:any) {
+   } catch (err:any) {
       const qType = type === 'stats' ? '(stats)' : `(${days}days)`;
-      console.log(`[ERROR] Search Console API Error for ${domainName} ${qType} : `, error?.response?.status, error?.response?.statusText);
-      return { error: true, errorMsg: `${error?.response?.status}: ${error?.response?.statusText}` };
+      const errorMsg = err?.response?.status && `${err?.response?.statusText}. ${err?.response?.data?.error_description}`;
+      console.log(`[ERROR] Search Console API Error for ${domainName} ${qType} : `, errorMsg || err?.code);
+      // console.log('SC ERROR :', err);
+      return { error: true, errorMsg: errorMsg || err?.code };
    }
 };
 
-export const fetchDomainSCData = async (domain:string): Promise<SCDomainDataType> => {
+/**
+ * The function fetches search console data for a given domain and returns it in a structured format.
+ * @param {DomainType} domain - The `domain` parameter is a Domain object that represents the domain for which we
+ * want to fetch search console data.
+ * @returns The function `fetchDomainSCData` is returning a Promise that resolves to an object of type
+ * `SCDomainDataType`.
+ */
+export const fetchDomainSCData = async (domain:DomainType, scAPI?: SCAPISettings): Promise<SCDomainDataType> => {
    const days = [3, 7, 30];
    const scDomainData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '', stats: [] };
-   if (domain) {
+   if (domain.domain && scAPI) {
+      const theDomain = domain;
       for (const day of days) {
-         const items = await fetchSearchConsoleData(domain, day);
+         const items = await fetchSearchConsoleData(theDomain, day, undefined, scAPI);
           scDomainData.lastFetched = new Date().toJSON();
          if (Array.isArray(items)) {
             if (day === 3) scDomainData.threeDays = items as SearchAnalyticsItem[];
@@ -83,16 +114,22 @@ export const fetchDomainSCData = async (domain:string): Promise<SCDomainDataType
             scDomainData.lastFetchError = items.errorMsg;
          }
       }
-      const stats = await fetchSearchConsoleData(domain, 30, 'stat');
+      const stats = await fetchSearchConsoleData(theDomain, 30, 'stat', scAPI);
       if (stats && Array.isArray(stats) && stats.length > 0) {
          scDomainData.stats = stats as SearchAnalyticsStat[];
       }
-      await updateLocalSCData(domain, scDomainData);
+      await updateLocalSCData(domain.domain, scDomainData);
    }
 
    return scDomainData;
 };
 
+/**
+ * The function takes a raw search console item and a domain name as input and returns a parsed search analytics item.
+ * @param {SearchAnalyticsRawItem} SCItem - The SCItem parameter is an object that represents a raw item from the Search Console API.
+ * @param {string} domainName - The `domainName` parameter is a string that represents the domain name of the website.
+ * @returns {SearchAnalyticsItem}.
+ */
 export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainName: string): SearchAnalyticsItem => {
    const { clicks = 0, impressions = 0, ctr = 0, position = 0 } = SCItem;
    const keyword = SCItem.keys[0];
@@ -104,6 +141,12 @@ export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainNam
    return { keyword, uid, device, country, clicks, impressions, ctr: ctr * 100, position, page };
 };
 
+/**
+ * The function integrates search console data with a keyword object and returns the updated keyword object with the search console data.
+ * @param {KeywordType} keyword - The `keyword` parameter is of type `KeywordType`, which is a custom type representing a keyword.
+ * @param {SCDomainDataType} SCData - SCData is an object that contains search analytics data for different time periods
+ * @returns {KeywordType}
+ */
 export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainDataType) : KeywordType => {
    const kuid = `${keyword.country.toLowerCase()}:${keyword.device}:${keyword.keyword.replaceAll(' ', '_')}`;
    const impressions:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 };
@@ -111,9 +154,9 @@ export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainData
    const ctr:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 };
    const position:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 };
 
-   const threeDaysData = SCData.threeDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
-   const SevenDaysData = SCData.sevenDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
-   const ThirdyDaysData = SCData.thirtyDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
+   const threeDaysData = SCData?.threeDays?.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
+   const SevenDaysData = SCData?.sevenDays?.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
+   const ThirdyDaysData = SCData?.thirtyDays?.find((item:SearchAnalyticsItem) => item.uid === kuid) || {};
    const totalData:any = { threeDays: threeDaysData, sevenDays: SevenDaysData, thirtyDays: ThirdyDaysData };
 
    Object.keys(totalData).forEach((dataKey) => {
@@ -136,18 +179,103 @@ export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainData
    return { ...keyword, scData: finalSCData };
 };
 
-export const readLocalSCData = async (domain:string): Promise<SCDomainDataType> => {
-   const filePath = `${process.cwd()}/data/SC_${domain}.json`;
-   const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch(async () => { await updateLocalSCData(domain); return '{}'; });
-   const domainSCData = JSON.parse(currentQueueRaw);
-   return domainSCData;
+/**
+ * Retrieves the Search Console API information for a given domain.
+ * @param {DomainType} domain - The `domain` parameter is of type `DomainType`, which represents a
+ * domain object. It likely contains information about a specific domain, such as its name, search
+ * console settings, etc.
+ * @returns an object of type `SCAPISettings`.
+ */
+export const getSearchConsoleApiInfo = async (domain: DomainType): Promise<SCAPISettings> => {
+   const scAPIData = { client_email: '', private_key: '' };
+   // Check if the Domain Has the API Data
+   const domainSCSettings = domain.search_console && JSON.parse(domain.search_console);
+   if (domainSCSettings && domainSCSettings.private_key) {
+      if (!domainSCSettings.private_key.includes('BEGIN PRIVATE KEY')) {
+         const cryptr = new Cryptr(process.env.SECRET as string);
+         scAPIData.client_email = domainSCSettings.client_email ? cryptr.decrypt(domainSCSettings.client_email) : '';
+         scAPIData.private_key = domainSCSettings.private_key ? cryptr.decrypt(domainSCSettings.private_key) : '';
+      } else {
+         scAPIData.client_email = domainSCSettings.client_email;
+         scAPIData.private_key = domainSCSettings.private_key;
+      }
+   }
+   // Check if the App Settings Has the API Data
+   if (!scAPIData?.private_key) {
+      const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
+      const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {};
+      const cryptr = new Cryptr(process.env.SECRET as string);
+      scAPIData.client_email = settings.search_console_client_email ? cryptr.decrypt(settings.search_console_client_email) : '';
+      scAPIData.private_key = settings.search_console_private_key ? cryptr.decrypt(settings.search_console_private_key) : '';
+   }
+   if (!scAPIData?.private_key && process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
+      scAPIData.client_email = process.env.SEARCH_CONSOLE_CLIENT_EMAIL;
+      scAPIData.private_key = process.env.SEARCH_CONSOLE_PRIVATE_KEY;
+   }
+
+   return scAPIData;
 };
 
+/**
+ * Checks if the provided domain level Google Search Console API info is valid.
+ * @param {DomainType} domain - The domain that represents the domain for which the SC API info is being checked.
+ * @returns an object of type `{ isValid: boolean, error: string }`.
+ */
+export const checkSerchConsoleIntegration = async (domain: DomainType): Promise<{ isValid: boolean, error: string }> => {
+   const res = { isValid: false, error: '' };
+   const { client_email = '', private_key = '' } = domain?.search_console ? JSON.parse(domain.search_console) : {};
+   const response = await fetchSearchConsoleData(domain, 3, undefined, { client_email, private_key });
+   if (Array.isArray(response)) { res.isValid = true; }
+   if ((response as SCDomainFetchError)?.errorMsg) { res.error = (response as SCDomainFetchError).errorMsg; }
+   return res;
+};
+
+/**
+ * The function reads and returns the domain-specific data stored in a local JSON file.
+ * @param {string} domain - The `domain` parameter is a string that represents the domain for which the SC data is being read.
+ * @returns {Promise<SCDomainDataType>}
+ */
+export const readLocalSCData = async (domain:string): Promise<SCDomainDataType|false> => {
+   try {
+      const filePath = `${process.cwd()}/data/SC_${domain.replaceAll('/', '-')}.json`;
+      const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch(async () => { await updateLocalSCData(domain); return '{}'; });
+      const domainSCData = JSON.parse(currentQueueRaw);
+      return domainSCData;
+   } catch (error) {
+      return false;
+   }
+};
+
+/**
+ * The function reads and returns the domain-specific data stored in a local JSON file.
+ * @param {string} domain - The `domain` parameter is a string that represents the domain for which the SC data will be written.
+ * @param {SCDomainDataType} scDomainData - an object that contains search analytics data for different time periods.
+ * @returns {Promise<SCDomainDataType|false>}
+ */
 export const updateLocalSCData = async (domain:string, scDomainData?:SCDomainDataType): Promise<SCDomainDataType|false> => {
-   const filePath = `${process.cwd()}/data/SC_${domain}.json`;
-   const emptyData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '' };
-   await writeFile(filePath, JSON.stringify(scDomainData || emptyData), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
-   return scDomainData || emptyData;
+   try {
+      const filePath = `${process.cwd()}/data/SC_${domain.replaceAll('/', '-')}.json`;
+      const emptyData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '' };
+      await writeFile(filePath, JSON.stringify(scDomainData || emptyData), { encoding: 'utf-8' }).catch((err) => { console.log(err); });
+      return scDomainData || emptyData;
+   } catch (error) {
+      return false;
+   }
+};
+
+/**
+ * The function removes the domain-specific Seach Console data stored in a local JSON file.
+ * @param {string} domain - The `domain` parameter is a string that represents the domain for which the SC data file will be removed.
+ * @returns {Promise<boolean>} - Returns true if file was removed, else returns false.
+ */
+export const removeLocalSCData = async (domain:string): Promise<boolean> => {
+   const filePath = `${process.cwd()}/data/SC_${domain.replaceAll('/', '-')}.json`;
+   try {
+      await unlink(filePath);
+      return true;
+   } catch (error) {
+      return false;
+   }
 };
 
 export default fetchSearchConsoleData;