chore(release): 2.0.7

Add error message, if returned search HTML does not contain required elements

.eslintrc.json

@@ -12,6 +12,10 @@
    "no-await-in-loop": "off",
    "arrow-body-style":"off",
    "max-len": ["error", {"code": 150, "ignoreComments": true, "ignoreUrls": true}],
+   "import/no-extraneous-dependencies": "off",
+   "no-unused-vars": "off",
+   "implicit-arrow-linebreak": "off",
+   "function-paren-newline": "off",
    "import/extensions": [
       "error",
       "ignorePackages",

.sequelizerc

@@ -0,0 +1,8 @@
+const path = require('path');
+
+module.exports = {
+  'config': path.resolve('database', 'config.js'),
+  'models-path': path.resolve('database', 'models'),
+  'seeders-path': path.resolve('database', 'seeders'),
+  'migrations-path': path.resolve('database', 'migrations')
+};

CHANGELOG.md

@@ -2,6 +2,293 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [2.0.7](https://github.com/towfiqi/serpbear/compare/v2.0.6...v2.0.7) (2025-02-23)
+
+
+### Bug Fixes
+
+* Resolves AdWords integration issue. ([36ed4cf](https://github.com/towfiqi/serpbear/commit/36ed4cf800c1fd0e3df4e807faaa1fdb863df5e4))
+* resolves broken CDN images. ([bf911b4](https://github.com/towfiqi/serpbear/commit/bf911b4e45b9007a05ce6399838da3276161c61d))
+
+### [2.0.6](https://github.com/towfiqi/serpbear/compare/v2.0.5...v2.0.6) (2024-11-15)
+
+
+### Bug Fixes
+
+* Ensures Docker build uses matching npm package versions from package.json ([4fef1a9](https://github.com/towfiqi/serpbear/commit/4fef1a9abc737da67ab1ea0c4efce8194890545e))
+* Resolves broken Docker build due to croner package version mismatch. ([aeed1f8](https://github.com/towfiqi/serpbear/commit/aeed1f8559e044bf658d930a22fa91f38cfedc6b)), closes [#247](https://github.com/towfiqi/serpbear/issues/247)
+* Resolves broken Proxy Scraper functionality. ([649f412](https://github.com/towfiqi/serpbear/commit/649f412303dd50127b3736740962863f735f76eb)), closes [#248](https://github.com/towfiqi/serpbear/issues/248)
+* Resolves Google Ads search volume data loading issue. ([12eac2b](https://github.com/towfiqi/serpbear/commit/12eac2b01235e9eae06882d6a2c50c793b890661))
+
+### [2.0.5](https://github.com/towfiqi/serpbear/compare/v2.0.4...v2.0.5) (2024-11-12)
+
+
+### Bug Fixes
+
+* Fixes broken scrape result issue for keywords with special characters. ([d9d7c63](https://github.com/towfiqi/serpbear/commit/d9d7c6347e99e35f1e48f20b1e8f999612d69a72)), closes [#221](https://github.com/towfiqi/serpbear/issues/221)
+* Fixes misaligned Keywords table UI content. ([faa3519](https://github.com/towfiqi/serpbear/commit/faa3519a29fc61ef8bd2ce9275a6674f1c7946e0))
+* Resolves "Add Domain" UI confusion. ([17fb2c4](https://github.com/towfiqi/serpbear/commit/17fb2c40cc6b57ab2fe6aeb940dececd1a83411f))
+* Resolves broken Scrapingrobot scraper on new installs. ([bc02c92](https://github.com/towfiqi/serpbear/commit/bc02c929ba7efd6b4b6a09495af7310c155a26bd)), closes [#243](https://github.com/towfiqi/serpbear/issues/243)
+
+### [2.0.4](https://github.com/towfiqi/serpbear/compare/v2.0.3...v2.0.4) (2024-11-10)
+
+
+### Bug Fixes
+
+* Fixes Docker build issue. ([1bef758](https://github.com/towfiqi/serpbear/commit/1bef7587cccada6df48cfa3d208bf123a5d00c30))
+
+### [2.0.3](https://github.com/towfiqi/serpbear/compare/v2.0.2...v2.0.3) (2024-11-10)
+
+
+### Features
+
+* Ability to keywords for both mobile and desktop at once. ([3786438](https://github.com/towfiqi/serpbear/commit/3786438662f015613330912f668c161f007f3eef)), closes [#60](https://github.com/towfiqi/serpbear/issues/60) [#66](https://github.com/towfiqi/serpbear/issues/66) [#199](https://github.com/towfiqi/serpbear/issues/199)
+* Adds ability to set Notification Email From name. ([b35d333](https://github.com/towfiqi/serpbear/commit/b35d333bfcdf0765bd105f74c8b324aec6d98b22)), closes [#222](https://github.com/towfiqi/serpbear/issues/222)
+* Adds the ability to show hide columns in tracked keywords table. ([d3e3760](https://github.com/towfiqi/serpbear/commit/d3e37605279c48f7a56b2ef5125c32a00b948e05)), closes [#224](https://github.com/towfiqi/serpbear/issues/224)
+* auto filter keywords if they already exist instead of throwing error. ([a09eb62](https://github.com/towfiqi/serpbear/commit/a09eb62f5a65f584c32ddb5c46018dc404d3e1f3)), closes [#244](https://github.com/towfiqi/serpbear/issues/244)
+* Displays Best position on mobile layout as well. ([a74338f](https://github.com/towfiqi/serpbear/commit/a74338fe15bee0fbc65a5197ee56db5d6a629bc8))
+* Displays keyword's best position in email notification. ([4c2f900](https://github.com/towfiqi/serpbear/commit/4c2f900d85e56b1994630942c9ea53d0fd0b4cdb))
+* Keywords Country filter now only shows relevant countries. ([a050536](https://github.com/towfiqi/serpbear/commit/a0505368140076d6626647993a41a5f4ef9db019))
+* Makes Content width a little wider. ([42c5e2b](https://github.com/towfiqi/serpbear/commit/42c5e2be0777a1bd72aa53bdf3c211075793a6e8))
+
+
+### Bug Fixes
+
+* Correct CTR calculation in InsightStats component ([232507e](https://github.com/towfiqi/serpbear/commit/232507e1ff7519cfc82f3818d3b757f2c427b52a))
+* Fixes incorrect position display in keyword detail view when position is above 100 ([01b1b7b](https://github.com/towfiqi/serpbear/commit/01b1b7b9e9c603e470217ad476dd54359d7d35b8))
+* Fixes missing keyword city value in exported csv file. ([f482884](https://github.com/towfiqi/serpbear/commit/f48288473e910c00e7c9178366a394e891bc47c7)), closes [#194](https://github.com/towfiqi/serpbear/issues/194)
+* Fixes missing Search Console data in Email notification when its integrated through App settings. ([040dab1](https://github.com/towfiqi/serpbear/commit/040dab15177d81874a6eb89f913a450f8a6f212d))
+* Resolves incorrect search trend graph in Ideas section. ([7597210](https://github.com/towfiqi/serpbear/commit/7597210ca2018b4abc82d441d2fc871e46295307)), closes [#219](https://github.com/towfiqi/serpbear/issues/219)
+* Resolves notification email's incorrect image size in some email clients. ([bc96dc7](https://github.com/towfiqi/serpbear/commit/bc96dc7de50844fbda19a89961c36cc65ac0b97b)), closes [#201](https://github.com/towfiqi/serpbear/issues/201)
+* update scraping robot typo in README ([c24b630](https://github.com/towfiqi/serpbear/commit/c24b63009c48c61218ea96511b11fe1a4f2ff239))
+
+### [2.0.2](https://github.com/towfiqi/serpbear/compare/v2.0.1...v2.0.2) (2024-03-13)
+
+
+### Bug Fixes
+
+* Resolves Broken Google Adwords Authentication in Docker containers. ([1d0a788](https://github.com/towfiqi/serpbear/commit/1d0a78881039b3ebd4b1e00fb067016886d49c31)), closes [#179](https://github.com/towfiqi/serpbear/issues/179)
+
+### [2.0.1](https://github.com/towfiqi/serpbear/compare/v2.0.0...v2.0.1) (2024-03-06)
+
+
+### Bug Fixes
+
+* Resolves broken doc links ([d48ae76](https://github.com/towfiqi/serpbear/commit/d48ae76103615ab82a32cbd8ffa62e27d17999de))
+* Resolves keyword loading issue in Docker instances. ([4a87d22](https://github.com/towfiqi/serpbear/commit/4a87d229fee7aa5e0ca8b2042c168465c7c5d67f)), closes [#178](https://github.com/towfiqi/serpbear/issues/178)
+
+## [2.0.0](https://github.com/towfiqi/serpbear/compare/v1.0.3...v2.0.0) (2024-03-05)
+
+
+### Features
+
+* Adds a Keyword Research Section. ([4d15989](https://github.com/towfiqi/serpbear/commit/4d15989b2832c3514fca18f3178a967c1fc9ad18))
+* Adds ability to pick existing tags when applying keyword tags. ([407ab8d](https://github.com/towfiqi/serpbear/commit/407ab8db831b26a5d45ccd7455c39f82fef1e438)), closes [#171](https://github.com/towfiqi/serpbear/issues/171)
+* Adds Google Adwords Integration to allow generating Keyword Ideas. ([5650645](https://github.com/towfiqi/serpbear/commit/5650645b58638e91bd5772415234fdda8cc8de1a))
+* Adds keyword search volume data feature for tracked keywords. ([2a1fc0e](https://github.com/towfiqi/serpbear/commit/2a1fc0e43d31f215b815a9d8b312da080209ae3a))
+* Adds the ability to view the changelog and displays the latest version number. ([bb4a684](https://github.com/towfiqi/serpbear/commit/bb4a6844b522067b69a007fda427d16d25bae3a3))
+
+
+### Bug Fixes
+
+* Resolves Domain keyword Ideas generation issue. ([252ae9a](https://github.com/towfiqi/serpbear/commit/252ae9aa84177909f5658d954ca1231c70cb9145))
+* Resolves minor UI bugs. ([50160f5](https://github.com/towfiqi/serpbear/commit/50160f5b234fc783802efae34db4a8f959745998))
+
+### [1.0.3](https://github.com/towfiqi/serpbear/compare/v1.0.2...v1.0.3) (2024-02-22)
+
+
+### Bug Fixes
+
+* Resolves App not reloading on Scraper setup. ([56ffbf5](https://github.com/towfiqi/serpbear/commit/56ffbf59d1e459a6c7229d141ccc6774dc8055d0))
+* Resolves large keywords breaking the keywords table ui ([724d3c8](https://github.com/towfiqi/serpbear/commit/724d3c8d4309c7bd4f40e9db980ad54f99023d35))
+* Resolves scraper not able to scrape some keywords correctly. ([9a7a43f](https://github.com/towfiqi/serpbear/commit/9a7a43f051387cacc13116c0a7c21716b54e539b))
+
+### [1.0.2](https://github.com/towfiqi/serpbear/compare/v1.0.1...v1.0.2) (2024-02-15)
+
+
+### Bug Fixes
+
+* Resolves Incorrect Position issue. ([0e64b95](https://github.com/towfiqi/serpbear/commit/0e64b95cd5303525535ea84a77181281d7f5618e)), closes [#164](https://github.com/towfiqi/serpbear/issues/164)
+
+### [1.0.1](https://github.com/towfiqi/serpbear/compare/v1.0.0...v1.0.1) (2024-02-13)
+
+
+### Bug Fixes
+
+* Resolves the app crash issue when there is no database. ([e5dd411](https://github.com/towfiqi/serpbear/commit/e5dd411aa9aef58ebb226f2b793a2632ab9069a7)), closes [#161](https://github.com/towfiqi/serpbear/issues/161) [#162](https://github.com/towfiqi/serpbear/issues/162)
+
+## [1.0.0](https://github.com/towfiqi/serpbear/compare/v0.3.4...v1.0.0) (2024-02-09)
+
+
+### Features
+
+* Adds Serper.dev integration ([b4ad69b](https://github.com/towfiqi/serpbear/commit/b4ad69baaa0f865938f8b0eace6732a9e6b1b381)), closes [#138](https://github.com/towfiqi/serpbear/issues/138)
+* Adds the ability for city level scraping for scapers that allow it. ([3719f21](https://github.com/towfiqi/serpbear/commit/3719f21d98d173219cef5656579fa0e5340ccdbf)), closes [#139](https://github.com/towfiqi/serpbear/issues/139) [#151](https://github.com/towfiqi/serpbear/issues/151)
+* adds the ability to add url as a domain. ([3c2a1b8](https://github.com/towfiqi/serpbear/commit/3c2a1b8a5b8a2a4a2179a5031582f8202c2e494a)), closes [#53](https://github.com/towfiqi/serpbear/issues/53) [#90](https://github.com/towfiqi/serpbear/issues/90) [#119](https://github.com/towfiqi/serpbear/issues/119)
+* Adds the Ability to set Search Console Property type via Domain Settings. ([b2e97b2](https://github.com/towfiqi/serpbear/commit/b2e97b2ebec380f0edf7ddc0640c2126eff006ac)), closes [#50](https://github.com/towfiqi/serpbear/issues/50)
+* Adds the ability to setup Search Console through the UI. ([f04b10c](https://github.com/towfiqi/serpbear/commit/f04b10cf6b065e3023965112a60e0aa702212a4b)), closes [#59](https://github.com/towfiqi/serpbear/issues/59) [#146](https://github.com/towfiqi/serpbear/issues/146)
+* Adds ValueSerp Integration. ([1041cb3](https://github.com/towfiqi/serpbear/commit/1041cb3c0bb69e9034696624e03433be28e83ac6)), closes [#105](https://github.com/towfiqi/serpbear/issues/105) [#106](https://github.com/towfiqi/serpbear/issues/106)
+
+
+### Bug Fixes
+
+* Resolves Keywords filter crashing issue. ([633ab2c](https://github.com/towfiqi/serpbear/commit/633ab2c467be5b7b86d4547ae0c59034e595a42d))
+* Resolves missing Keyword Loading Spinner issue. ([dbf540c](https://github.com/towfiqi/serpbear/commit/dbf540cfdb16ddb02c9d26618e3680d34799f57f))
+
+### [0.3.4](https://github.com/towfiqi/serpbear/compare/v0.3.3...v0.3.4) (2024-01-15)
+
+
+### Features
+
+* adds ability to add multiple domains at once. ([faa88c9](https://github.com/towfiqi/serpbear/commit/faa88c92542194f19b5cfe2b5cfd07d7d4f7ee46))
+* Adds the ability to show/hide Keys & Passwords in Settings Panel ([c897a52](https://github.com/towfiqi/serpbear/commit/c897a525509baf5b9e8df18d82f5e87aec64f66e))
+
+
+### Bug Fixes
+
+* fixes local SC data not being removed on deleting domain. ([cca9f95](https://github.com/towfiqi/serpbear/commit/cca9f95358b2d3ea06edb33595cdbf616a175469))
+* Resolves incorrect keyword average SC data values in Tracker ([e166b58](https://github.com/towfiqi/serpbear/commit/e166b588aa6c8db55d61b5bc13db66514575c745))
+* resolves newly added Domain's Update time rendering issue ([df3a738](https://github.com/towfiqi/serpbear/commit/df3a738788fa957e7246a0feefe395a9eadd5baf))
+
+### [0.3.3](https://github.com/towfiqi/serpbear/compare/v0.3.2...v0.3.3) (2023-11-12)
+
+
+### Features
+
+* Adds ability to visit pages from Insight tab ([60c68bd](https://github.com/towfiqi/serpbear/commit/60c68bd339db7aeed35aea035dd21691702ffee3))
+* Domains now show their favicon. ([2339e31](https://github.com/towfiqi/serpbear/commit/2339e31af9e90bf918f5bcd4f23114f38cef0313)), closes [#130](https://github.com/towfiqi/serpbear/issues/130)
+* Shows total keywords count in domains page ([fbd23ed](https://github.com/towfiqi/serpbear/commit/fbd23ede256062c72ec2f7e3983a0a02f0240725))
+
+
+### Bug Fixes
+
+* Resolves Website Thumbnail missing issue ([4a60271](https://github.com/towfiqi/serpbear/commit/4a60271cac1209dc02748c4d31943bb21c9ecaf2)), closes [#131](https://github.com/towfiqi/serpbear/issues/131)
+
+### [0.3.2](https://github.com/towfiqi/serpbear/compare/v0.3.1...v0.3.2) (2023-11-09)
+
+
+### Bug Fixes
+
+* Resolves issue with adding long tld emails ([9b9b74a](https://github.com/towfiqi/serpbear/commit/9b9b74af4c249e27458d29ba052e96ab2db8b640)), closes [#127](https://github.com/towfiqi/serpbear/issues/127)
+
+### [0.3.1](https://github.com/towfiqi/serpbear/compare/v0.3.0...v0.3.1) (2023-11-04)
+
+
+### Bug Fixes
+
+* Removes dev files from docker volumes ([454454a](https://github.com/towfiqi/serpbear/commit/454454a422bab4d37a2d43ad95868e293a97b88e))
+* Updates vulnerable dependencies ([97dd0b1](https://github.com/towfiqi/serpbear/commit/97dd0b131be4cec73d07f35062334dd1881f0013))
+
+## [0.3.0](https://github.com/towfiqi/serpbear/compare/v0.2.6...v0.3.0) (2023-11-03)
+
+
+### Features
+
+* Adds ability to disable/clear retry queue for failed keywords ([dc3c7a7](https://github.com/towfiqi/serpbear/commit/dc3c7a722b18248115969c51f2495ccf1c43926d))
+* Adds ability to search w/o case sensitivity ([4748ffc](https://github.com/towfiqi/serpbear/commit/4748ffc382161c5d861b8d43e8eba466a031e2bc)), closes [#115](https://github.com/towfiqi/serpbear/issues/115)
+* Displays the Best position of the keywords ([fc183d2](https://github.com/towfiqi/serpbear/commit/fc183d246d55e0eecf43c91f6da8a59192e8e771)), closes [#89](https://github.com/towfiqi/serpbear/issues/89)
+* Refresh All feature now shows update real-time ([1d6b2be](https://github.com/towfiqi/serpbear/commit/1d6b2be95aa133b7998f5cf098f15aa32f5badd2))
+* Remembers last selected coutry ([d3d336f](https://github.com/towfiqi/serpbear/commit/d3d336fa71cc789624b10f3cdd1a2b5983053e6f)), closes [#101](https://github.com/towfiqi/serpbear/issues/101)
+
+
+### Bug Fixes
+
+*  Resolves missing keyword scrape spinner issue ([f57bca2](https://github.com/towfiqi/serpbear/commit/f57bca23daa3fe888af4c19a681dcec6b6100d83))
+* Cron stopped on failing to parse failed queue ([8a949ce](https://github.com/towfiqi/serpbear/commit/8a949ce4c078ff377e91a95c4b86ef2b15dae88b)), closes [#116](https://github.com/towfiqi/serpbear/issues/116)
+* Fixes import order error in some instances. ([be80ed7](https://github.com/towfiqi/serpbear/commit/be80ed7ef3dd0a315c5ad67d17e61a4797dc274c)), closes [#114](https://github.com/towfiqi/serpbear/issues/114)
+* Fixes issue with adding hyphenated subdomains. ([c0470cf](https://github.com/towfiqi/serpbear/commit/c0470cfa9d0dac86317c886065b461cfe82ffb16))
+* Fixes the weekly cron day issue. ([392122a](https://github.com/towfiqi/serpbear/commit/392122a7101683342830e900c6f0c39f9272bb34)), closes [#118](https://github.com/towfiqi/serpbear/issues/118)
+* Fxies special character keyword scrape issue. ([9feff13](https://github.com/towfiqi/serpbear/commit/9feff13f18a4d72203dde694a147831f990b37fb)), closes [#113](https://github.com/towfiqi/serpbear/issues/113) [#122](https://github.com/towfiqi/serpbear/issues/122)
+
+### [0.2.6](https://github.com/towfiqi/serpbear/compare/v0.2.5...v0.2.6) (2023-03-29)
+
+
+### Features
+
+* Add option to Delay Between scrapes. ([0a83924](https://github.com/towfiqi/serpbear/commit/0a83924ffe2243c52849c167c6c15d9688ff1dc7)), closes [#87](https://github.com/towfiqi/serpbear/issues/87)
+* Integrates Space Serp. ([0538a8c](https://github.com/towfiqi/serpbear/commit/0538a8c01601d2f6365848580591a248528e67c7))
+
+
+### Bug Fixes
+
+* **components:** fix typo "Goolge" -> "Google" ([dce7c41](https://github.com/towfiqi/serpbear/commit/dce7c412e813fc845973f36ad1c9fa91df4a6611))
+* Fixes first Keryword Error cut off issue. ([d950515](https://github.com/towfiqi/serpbear/commit/d9505158c439a924a1c86eb8243faf2a15bed43e))
+* Fixes lags when tracking thousands of keywords ([9757fde](https://github.com/towfiqi/serpbear/commit/9757fde02ec83405546733381104c54ed6510681)), closes [#88](https://github.com/towfiqi/serpbear/issues/88)
+
+### [0.2.5](https://github.com/towfiqi/serpbear/compare/v0.2.4...v0.2.5) (2023-03-07)
+
+
+### Features
+
+* Adds current App version Number in Footer. ([b83df5f](https://github.com/towfiqi/serpbear/commit/b83df5f3dbd64db657d31f0526438e7165e1b475))
+* Adds Keyword Scraping Interval Settings. ([3b6d034](https://github.com/towfiqi/serpbear/commit/3b6d034d6f7da0b4259070220fffff44184dd680)), closes [#81](https://github.com/towfiqi/serpbear/issues/81) [#76](https://github.com/towfiqi/serpbear/issues/76)
+
+
+### Bug Fixes
+
+* Fixes Broken Image thumbnail loading issue. ([5dd366b](https://github.com/towfiqi/serpbear/commit/5dd366b91e2a94e658bf5250a8a0fa64c09e1c11))
+* Settings Update Toast was not showing up. ([b9d58a7](https://github.com/towfiqi/serpbear/commit/b9d58a721df12f3f34220a3ae5da6897e23c83ec))
+
+### [0.2.4](https://github.com/towfiqi/serpbear/compare/v0.2.3...v0.2.4) (2023-02-15)
+
+
+### Features
+
+* Keyword ranking pages can now be clicked. ([c5af94a](https://github.com/towfiqi/serpbear/commit/c5af94a1469713ed4092253d26953ee0ed28c25d))
+
+
+### Bug Fixes
+
+* Fixes broken Login on windows ([c406588](https://github.com/towfiqi/serpbear/commit/c406588953035e4177a64011c13eb0e3aedffe89))
+* Fixes Node Cron memory leak issue. ([3c48d13](https://github.com/towfiqi/serpbear/commit/3c48d130b6f229a4ac27ec43ef1ea3a6640cecf6))
+
+### [0.2.3](https://github.com/towfiqi/serpbear/compare/v0.2.2...v0.2.3) (2023-01-12)
+
+
+### Features
+
+* Ability to tag multiple keywords at once ([9e9dad7](https://github.com/towfiqi/serpbear/commit/9e9dad7631691b2a836fdd4c522b1f933b17e285)), closes [#54](https://github.com/towfiqi/serpbear/issues/54)
+* Set USERNAME as well as USER variable ([b50733d](https://github.com/towfiqi/serpbear/commit/b50733defc2c06e0f92ca3e88fd1f74684eee9c0))
+
+
+### Bug Fixes
+
+* Fixes Position and View Sort. ([8139e39](https://github.com/towfiqi/serpbear/commit/8139e399c13ab8be767facef9a19c67dec06ed64)), closes [#46](https://github.com/towfiqi/serpbear/issues/46)
+* Fixes wrong CTR value for Search Console Data ([cb24696](https://github.com/towfiqi/serpbear/commit/cb24696a1f47b02a11c68cd1c673ea8b1bacd144)), closes [#48](https://github.com/towfiqi/serpbear/issues/48)
+* Mobile Keyword Scraping not working. ([a1108d2](https://github.com/towfiqi/serpbear/commit/a1108d240ea38ab0886ef3722b0c937ec5a45591)), closes [#58](https://github.com/towfiqi/serpbear/issues/58)
+* ScrapingAnt Mobile Keyword Scrape not working ([acc0b39](https://github.com/towfiqi/serpbear/commit/acc0b39d80d4f9371967a0d425ed205c5d866eea))
+
+### [0.2.2](https://github.com/towfiqi/serpbear/compare/v0.2.1...v0.2.2) (2022-12-25)
+
+
+### Bug Fixes
+
+* Fixes bug that prevents Saving API settings ([123ad81](https://github.com/towfiqi/serpbear/commit/123ad81dae10aa28848148d0f3da5cf1f7de7c57)), closes [#45](https://github.com/towfiqi/serpbear/issues/45)
+
+### [0.2.1](https://github.com/towfiqi/serpbear/compare/v0.2.0...v0.2.1) (2022-12-24)
+
+## [0.2.0](https://github.com/towfiqi/serpbear/compare/v0.1.7...v0.2.0) (2022-12-21)
+
+
+### Features
+
+* Adds better error logging for debugging issues ([9b71f84](https://github.com/towfiqi/serpbear/commit/9b71f8400bc17b75722b93cbe745543f6b30814a))
+* Highlights tracked keywords in Discovery tab ([ee32435](https://github.com/towfiqi/serpbear/commit/ee32435d05c2a2ec6d446cd00e28058f07eb1ad4))
+* Integrate SerpApi ([ad6a354](https://github.com/towfiqi/serpbear/commit/ad6a354cb93bc6584d71dd1216a8a03d8dba505b))
+* integrates Google Search Console. ([49b4769](https://github.com/towfiqi/serpbear/commit/49b4769528d18e34c16386b73dfb662e7a9f45a0))
+
+
+### Bug Fixes
+
+* Ability to add SMTP without user/pass. ([671f89e](https://github.com/towfiqi/serpbear/commit/671f89e492b0f45d63ae7575c7d4970252c11296)), closes [#30](https://github.com/towfiqi/serpbear/issues/30)
+* backend error on addind new domain ([c2b6328](https://github.com/towfiqi/serpbear/commit/c2b63280cb9d66b565dc51eb69ee960710ace895))
+* Backend error on loading the domains page. ([e3bd5b9](https://github.com/towfiqi/serpbear/commit/e3bd5b9c0735939c6b06e9762a3ad041b8b05d6e))
+* Email Notification was not being sent. ([0fdb43c](https://github.com/towfiqi/serpbear/commit/0fdb43c0a53460cd35daabc4703d26cb11db9601))
+* Fixes Docker Deployment failure after the SC integration. ([b0bfba4](https://github.com/towfiqi/serpbear/commit/b0bfba440464f8fc7c31609c202e01416a41702d))
+* hides Search Console Stats if its not connected ([b740ef3](https://github.com/towfiqi/serpbear/commit/b740ef337bbfb43f63528cac891d4cb254318dc7))
+* Keyword Detail View's broken Search Result ([6a1f1d4](https://github.com/towfiqi/serpbear/commit/6a1f1d4adff89fc718c0f2ffe52a59ab15ad6c80))
+* Minor UI Issues. ([89824ec](https://github.com/towfiqi/serpbear/commit/89824ece2349b510fa0b7d87b33cacd2c88efc95))
+
 ### [0.1.7](https://github.com/towfiqi/serpbear/compare/v0.1.6...v0.1.7) (2022-12-08)
 
 

Dockerfile

@@ -1,5 +1,6 @@
-FROM node:lts-alpine AS deps
-
+FROM node:22.11.0-alpine3.20 AS deps
+ENV NPM_VERSION=10.3.0
+RUN npm install -g npm@"${NPM_VERSION}"
 WORKDIR /app
 
 COPY package.json ./
@@ -7,15 +8,22 @@ RUN npm install
 COPY . .
 
 
-FROM node:lts-alpine AS builder
+FROM node:22.11.0-alpine3.20 AS builder
 WORKDIR /app
+ENV NPM_VERSION=10.3.0
+RUN npm install -g npm@"${NPM_VERSION}"
 COPY --from=deps /app ./
+RUN rm -rf /app/data
+RUN rm -rf /app/__tests__
+RUN rm -rf /app/__mocks__
 RUN npm run build
 
 
-FROM node:lts-alpine AS runner
+FROM node:22.11.0-alpine3.20 AS runner
 WORKDIR /app
-ENV NODE_ENV production
+ENV NPM_VERSION=10.3.0
+RUN npm install -g npm@"${NPM_VERSION}"
+ENV NODE_ENV=production
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 RUN set -xe && mkdir -p /app/data && chown nextjs:nodejs /app/data
@@ -27,13 +35,17 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 # setup the cron
 COPY --from=builder --chown=nextjs:nodejs /app/cron.js ./
 COPY --from=builder --chown=nextjs:nodejs /app/email ./email
+COPY --from=builder --chown=nextjs:nodejs /app/database ./database
+COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc
+COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh
 RUN rm package.json
 RUN npm init -y 
-RUN npm i cryptr dotenv node-cron
+RUN npm i cryptr@6.0.3 dotenv@16.0.3 croner@9.0.0 @googleapis/searchconsole@1.0.5 sequelize-cli@6.6.2 @isaacs/ttlcache@1.4.1
 RUN npm i -g concurrently
 
 USER nextjs
 
 EXPOSE 3000
 
+ENTRYPOINT ["/app/entrypoint.sh"]
 CMD ["concurrently","node server.js", "node cron.js"]

README.md

@@ -1,46 +1,60 @@
-![SerpBear](https://i.imgur.com/0S2zIH3.png) 
+![SerpBear](https://i.imgur.com/0S2zIH3.png)
+
 # SerpBear
 
-![Codacy Badge](https://app.codacy.com/project/badge/Grade/7e7a0030c3f84c6fb56a3ce6273fbc1d) ![GitHub](https://img.shields.io/github/license/towfiqi/serpbear) ![GitHub package.json version](https://img.shields.io/github/package-json/v/towfiqi/serpbear) ![Docker Pulls](https://img.shields.io/docker/pulls/towfiqi/serpbear)
+![Codacy Badge](https://app.codacy.com/project/badge/Grade/7e7a0030c3f84c6fb56a3ce6273fbc1d) ![GitHub](https://img.shields.io/github/license/towfiqi/serpbear) ![GitHub package.json version](https://img.shields.io/github/package-json/v/towfiqi/serpbear) ![Docker Pulls](https://img.shields.io/docker/pulls/towfiqi/serpbear) [![StandWithPalestine](https://raw.githubusercontent.com/Safouene1/support-palestine-banner/master/StandWithPalestine.svg)](https://www.youtube.com/watch?v=bjtDsd0g468&rco=1)
 
 #### [Documentation](https://docs.serpbear.com/) | [Changelog](https://github.com/towfiqi/serpbear/blob/main/CHANGELOG.md) | [Docker Image](https://hub.docker.com/r/towfiqi/serpbear)
 
-SerpBear is an Open Source Search Engine Position Tracking App. It allows you to track your website's keyword positions in Google and get notified of their positions.
+SerpBear is an Open Source Search Engine Position Tracking and Keyword Research App. It allows you to track your website's keyword positions in Google and get notified of their position change.
 
-![Easy to Use Search Engine Rank Tracker](https://i.imgur.com/bRzpmCK.gif)
+![Easy to Use Search Engine Rank Tracker](https://serpbear.b-cdn.net/serpbear_readme_v2.gif)
 
 #### Features
--   **Unlimited Keywords:** Add unlimited domains and unlimited keywords to track their SERP.
--   **Email Notification:** Get notified of your keyword position changes daily/weekly/monthly through email.
--   **SERP API:** SerpBear comes with built-in API that you can use for your marketing & data reporting tools.
--   **Mobile App:** Add the PWA app to your mobile for a better mobile experience. 
--   **Zero Cost to RUN:** Run the App on mogenius.com or Fly.io for free.
+
+- **Unlimited Keywords:** Add unlimited domains and unlimited keywords to track their SERP.
+- **Email Notification:** Get notified of your keyword position changes daily/weekly/monthly through email.
+- **SERP API:** SerpBear comes with built-in API that you can use for your marketing & data reporting tools.
+- **Keyword Research:** Ability to research keywords and auto-generate keyword ideas from your tracked website's content by integrating your Google Ads test account.
+- **Google Search Console Integration:** Get the actual visit count, impressions & more for Each keyword.
+- **Mobile App:** Add the PWA app to your mobile for a better mobile experience.
+- **Zero Cost to RUN:** Run the App on mogenius.com or Fly.io for free.
 
 #### How it Works
-The App uses third party website scrapers like ScrapingAnt, ScrapingRobot or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword.
+
+The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi, HasData or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword.
+
+The Keyword Research and keyword generation feature works by integrating your Google Ads test accounts into SerpBear. You can also view the added keyword's monthly search volume data once you [integrate Google Ads](https://docs.serpbear.com/miscellaneous/integrate-google-ads).
+
+When you [integrate Google Search Console](https://docs.serpbear.com/miscellaneous/integrate-google-search-console), the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages.you will be able to view the actual visits count from Google Search for the tracked keywords.
 
 #### Getting Started
--   **Step 1:** Deploy & Run the App.
--   **Step 2:** Access your App and Login.
--   **Step 3:** Add your First domain.
--   **Step 4:** Get an free API key from either ScrapingAnt or ScrapingRobot. Skip if you want to use Proxy ips.
--   **Step 5:** Setup the Scraping API/Proxy from the App's Settings interface.
--   **Step 6:** Add your keywords and start tracking.
--   **Step 7:** Optional. From the Settings panel, setup SMTP details to get notified of your keywords positions through email. You can use ElasticEmail and Sendpulse SMTP services that are free.  
 
-#### Compare SerpBear with other SERP tracking services
+- **Step 1:** Deploy & Run the App.
+- **Step 2:** Access your App and Login.
+- **Step 3:** Add your First domain.
+- **Step 4:** Get a free API key from ScrapingRobot or select a paid provider (see below) . Skip if you want to use Proxy ips.
+- **Step 5:** Setup the Scraping API/Proxy from the App's Settings interface.
+- **Step 6:** Add your keywords and start tracking.
+- **Step 7:** Optional. From the Settings panel, setup SMTP details to get notified of your keywords positions through email. You can use ElasticEmail and Sendpulse SMTP services that are free.
 
-|Service  | Cost | SERP Lookup | API |
-|--|--|--|--|
-| SerpBear | Free* | Unlimited* | Yes |
-| ranktracker.com | $18/mo| 3,000/mo| No |
-| SerpWatch.io | $29/mo | 7500/mo | Yes |
-| Serpwatcher.com | $49/mo| 3000/mo | No |
-| whatsmyserp.com | $49/mo| 30,000/mo| No |
-| serply.io | $49/mo | 5000/mo | Yes |
+#### SerpBear Integrates with popular SERP scraping services
 
-(*) Free upto a limit. If you are using ScrapingAnt you can lookup 10,000 times per month for free.
+If you don't want to use proxies, you can use third party Scraping services to scrape Google Search results.
 
-**Stack**
--   Next.js for Frontend & Backend.  
--   Sqlite for Database.
+
+| Service           | Cost          | SERP Lookup    | API |
+| ----------------- | ------------- | -------------- | --- |
+| scrapingrobot.com | Free          | 5000/mo        | Yes |
+| serply.io         | $49/mo        | 5000/mo        | Yes |
+| serpapi.com       | From $50/mo   | From 5,000/mo  | Yes |
+| spaceserp.com     | $59/lifetime  | 15,000/mo      | Yes |
+| SearchApi.io      | From $40/mo   | From 10,000/mo | Yes |
+| valueserp.com     | Pay As You Go | $2.50/1000 req | No  |
+| serper.dev        | Pay As You Go | $1.00/1000 req | No  |
+| hasdata.com       | From $29/mo   | From 10,000/mo | Yes |
+
+**Tech Stack**
+
+- Next.js for Frontend & Backend.
+- Sqlite for Database.

__mocks__/data.ts

@@ -2,10 +2,11 @@ export const dummyDomain = {
    ID: 1,
    domain: 'compressimage.io',
    slug: 'compressimage-io',
-   keywordCount: 0,
+   keywordCount: 10,
+   avgPosition: 24,
    lastUpdated: '2022-11-11T10:00:32.243',
    added: '2022-11-11T10:00:32.244',
-   tags: [],
+   tags: '',
    notification: true,
    notification_interval: 'daily',
    notification_emails: '',
@@ -21,6 +22,7 @@ export const dummyKeywords = [
        lastUpdated: '2022-11-15T10:49:53.113',
        added: '2022-11-11T10:01:06.951',
        position: 19,
+       volume: 10000,
        history: {
            '2022-11-11': 21,
            '2022-11-12': 24,
@@ -33,7 +35,7 @@ export const dummyKeywords = [
        lastResult: [],
        sticky: false,
        updating: false,
-       lastUpdateError: 'false',
+       lastUpdateError: false as false,
    },
    {
        ID: 2,
@@ -44,6 +46,7 @@ export const dummyKeywords = [
        lastUpdated: '2022-11-15T10:49:53.119',
        added: '2022-11-15T10:01:06.951',
        position: 29,
+       volume: 1200,
        history: {
            '2022-11-11': 33,
            '2022-11-12': 34,
@@ -56,6 +59,24 @@ export const dummyKeywords = [
        lastResult: [],
        sticky: false,
        updating: false,
-       lastUpdateError: 'false',
+       lastUpdateError: false as false,
    },
 ];
+
+export const dummySettings = {
+   scaping_api: '',
+   scraper_type: 'none',
+   notification_interval: 'never',
+   notification_email: '',
+   notification_email_from: '',
+   notification_email_from_name: 'SerpBear',
+   smtp_server: '',
+   smtp_port: '',
+   smtp_username: '',
+   smtp_password: '',
+   scrape_retry: false,
+   search_console_integrated: false,
+   screenshot_key: '',
+   available_scapers: [],
+   failed_queue: [],
+};

__mocks__/utils.tsx

@@ -1,22 +1,26 @@
 import { render } from '@testing-library/react';
-import { rest } from 'msw';
+import { http } from 'msw';
 import * as React from 'react';
 import { QueryClient, QueryClientProvider } from 'react-query';
 
 export const handlers = [
-    rest.get(
+    http.get(
         '*/react-query',
-        (req, res, ctx) => {
-            return res(
-                ctx.status(200),
-                ctx.json({
+        ({ request, params }) => {
+            return new Response(
+                JSON.stringify({
                     name: 'mocked-react-query',
                 }),
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                },
             );
         },
     ),
 ];
-
 const createTestQueryClient = () => new QueryClient({
     defaultOptions: {
         queries: {

__test__/components/Modal.test.tsx

@@ -1,35 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import React from 'react';
-import Modal from '../../components/common/Modal';
-
-// jest.mock('React', () => ({
-//    ...jest.requireActual('React'),
-//    useEffect: jest.fn(),
-// }));
-// const mockedUseEffect = useEffect as jest.Mock<any>;
-
-// jest.mock('../../components/common/Icon', () => () => <div data-testid="icon"/>);
-
-describe('Modal Component', () => {
-   it('Renders without crashing', async () => {
-       render(<Modal closeModal={() => console.log() }><div></div></Modal>);
-      //  mockedUseEffect.mock.calls[0]();
-       expect(document.querySelector('.modal')).toBeInTheDocument();
-   });
-//    it('Sets up the escapae key shortcut', async () => {
-//       render(<Modal closeModal={() => console.log() }><div></div></Modal>);
-//       expect(mockedUseEffect).toBeCalled();
-//   });
-   it('Displays the Given Content', async () => {
-      render(<Modal closeModal={() => console.log() }>
-        <div>
-           <h1>Hello Modal!!</h1>
-        </div>
-      </Modal>);
-      expect(await screen.findByText('Hello Modal!!')).toBeInTheDocument();
-   });
-   it('Renders Modal Title', async () => {
-      render(<Modal closeModal={() => console.log() } title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
-      expect(await screen.findByText('Sample Modal Title')).toBeInTheDocument();
-   });
-});

__test__/components/Sidebar.test.tsx

@@ -1,11 +0,0 @@
-import { render, screen } from '@testing-library/react';
-import Sidebar from '../../components/common/Sidebar';
-
-describe('Sidebar Component', () => {
-   it('renders without crashing', async () => {
-       render(<Sidebar domains={[]} showAddModal={() => console.log() } />);
-       expect(
-           await screen.findByText('SerpBear'),
-       ).toBeInTheDocument();
-   });
-});

__test__/hooks/domains.tests.tsx

@@ -1,21 +0,0 @@
-import { waitFor } from '@testing-library/react';
-// import { useFetchDomains } from '../../services/domains';
-// import { createWrapper } from '../utils';
-
-jest.mock('next/router', () => ({
-   useRouter: () => ({
-     query: { slug: 'compressimage-io' },
-     push: (link:string) => { console.log('Pushed', link); },
-   }),
-}));
-
-describe('DomainHooks', () => {
-   it('useFetchDomains should fetch the Domains', async () => {
-      // const { result } = renderHook(() => useFetchDomains(), { wrapper: createWrapper() });
-      const result = { current: { isSuccess: false, data: '' } };
-      await waitFor(() => {
-         console.log('result.current: ', result.current.data);
-         return expect(result.current.isSuccess).toBe(true);
-      });
-   });
-});

__tests__/components/DomainItem.test.tsx

@@ -0,0 +1,35 @@
+import { fireEvent, render } from '@testing-library/react';
+import DomainItem from '../../components/domains/DomainItem';
+import { dummyDomain } from '../../__mocks__/data';
+
+const updateThumbMock = jest.fn();
+const domainItemProps = {
+   domain: dummyDomain,
+   selected: false,
+   isConsoleIntegrated: false,
+   thumb: '',
+   updateThumb: updateThumbMock,
+};
+
+describe('DomainItem Component', () => {
+   it('renders without crashing', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      expect(container.querySelector('.domItem')).toBeInTheDocument();
+   });
+   it('renders keywords count', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const domStatskeywords = container.querySelector('.dom_stats div:nth-child(1)');
+      expect(domStatskeywords?.textContent).toBe('Keywords10');
+   });
+   it('renders avg position', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const domStatsAvg = container.querySelector('.dom_stats div:nth-child(2)');
+      expect(domStatsAvg?.textContent).toBe('Avg position24');
+   });
+   it('updates domain thumbnail on relevant button click', async () => {
+      const { container } = render(<DomainItem {...domainItemProps} />);
+      const reloadThumbbBtn = container.querySelector('.domain_thumb button');
+      if (reloadThumbbBtn) fireEvent.click(reloadThumbbBtn);
+      expect(updateThumbMock).toHaveBeenCalledWith(dummyDomain.domain);
+   });
+});

__tests__/components/Keyword.test.tsx

@@ -1,8 +1,14 @@
 import { fireEvent, render, screen } from '@testing-library/react';
 import Keyword from '../../components/keywords/Keyword';
-import { dummyKeywords } from '../data';
+import { dummyKeywords } from '../../__mocks__/data';
 
-const keywordFunctions = {
+const keywordProps = {
+   keywordData: dummyKeywords[0],
+   selected: false,
+   index: 0,
+   showSCData: false,
+   scDataType: '',
+   style: {},
    refreshkeyword: jest.fn(),
    favoriteKeyword: jest.fn(),
    removeKeyword: jest.fn(),
@@ -10,35 +16,37 @@ const keywordFunctions = {
    manageTags: jest.fn(),
    showKeywordDetails: jest.fn(),
 };
-
+jest.mock('react-chartjs-2', () => ({
+   Line: () => null,
+ }));
 describe('Keyword Component', () => {
    it('renders without crashing', async () => {
-       render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+       render(<Keyword {...keywordProps} />);
        expect(await screen.findByText('compress image')).toBeInTheDocument();
    });
    it('Should Render Position Correctly', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_position');
       expect(positionElement?.childNodes[0].nodeValue).toBe('19');
    });
    it('Should Display Position Change arrow', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_position i');
       expect(positionElement?.textContent).toBe('▲ 1');
    });
    it('Should Display the SERP Page URL', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+      render(<Keyword {...keywordProps} />);
       const positionElement = document.querySelector('.keyword_url');
       expect(positionElement?.textContent).toBe('/');
    });
    it('Should Display the Keyword Options on dots Click', async () => {
-      render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
-      const button = document.querySelector('.keyword .keyword_dots');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      const { container } = render(<Keyword {...keywordProps} />);
+      const button = container.querySelector('.keyword_dots');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.keyword_options')).toBeVisible();
    });
    // it('Should favorite Keywords', async () => {
-   //    render(<Keyword keywordData={dummyKeywords[0]} {...keywordFunctions} selected={false} />);
+   //    render(<Keyword {...keywordProps} />);
    //    const button = document.querySelector('.keyword .keyword_dots');
    //    if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
    //    const option = document.querySelector('.keyword .keyword_options li:nth-child(1) a');

__tests__/components/Modal.test.tsx

@@ -0,0 +1,28 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import Modal from '../../components/common/Modal';
+
+const closeModalMock = jest.fn();
+describe('Modal Component', () => {
+   it('Renders without crashing', async () => {
+       render(<Modal closeModal={closeModalMock }><div></div></Modal>);
+       expect(document.querySelector('.modal')).toBeInTheDocument();
+   });
+   it('Displays the Given Content', async () => {
+      render(<Modal closeModal={closeModalMock}>
+        <div>
+           <h1>Hello Modal!!</h1>
+        </div>
+      </Modal>);
+      expect(await screen.findByText('Hello Modal!!')).toBeInTheDocument();
+   });
+   it('Renders Modal Title', async () => {
+      render(<Modal closeModal={closeModalMock} title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
+      expect(await screen.findByText('Sample Modal Title')).toBeInTheDocument();
+   });
+   it('Closes the modal on close button click', async () => {
+      const { container } = render(<Modal closeModal={closeModalMock} title="Sample Modal Title"><p>Some Modal Content</p></Modal>);
+      const closeBtn = container.querySelector('.modal-close');
+      if (closeBtn) fireEvent.click(closeBtn);
+      expect(closeModalMock).toHaveBeenCalled();
+   });
+});

__tests__/components/Sidebar.test.tsx

@@ -0,0 +1,23 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import Sidebar from '../../components/common/Sidebar';
+import { dummyDomain } from '../../__mocks__/data';
+
+const addDomainMock = jest.fn();
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+
+describe('Sidebar Component', () => {
+   it('renders without crashing', async () => {
+       render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+       expect(screen.getByText('SerpBear')).toBeInTheDocument();
+   });
+   it('renders domain list', async () => {
+      render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+      expect(screen.getByText('compressimage.io')).toBeInTheDocument();
+   });
+   it('calls showAddModal on Add Domain button click', async () => {
+      render(<Sidebar domains={[dummyDomain]} showAddModal={addDomainMock} />);
+      const addDomainBtn = screen.getByTestId('add_domain');
+      fireEvent.click(addDomainBtn);
+      expect(addDomainMock).toHaveBeenCalledWith(true);
+   });
+});

__tests__/components/Topbar.test.tsx

@@ -1,9 +1,15 @@
 import { render, screen } from '@testing-library/react';
 import TopBar from '../../components/common/TopBar';
 
+jest.mock('next/router', () => ({
+   useRouter: () => ({
+      pathname: '/',
+   }),
+}));
+
 describe('TopBar Component', () => {
    it('renders without crashing', async () => {
-       render(<TopBar showSettings={() => console.log() } />);
+       render(<TopBar showSettings={jest.fn} showAddModal={jest.fn} />);
        expect(
            await screen.findByText('SerpBear'),
        ).toBeInTheDocument();

__tests__/hooks/domains.test.tsx

@@ -0,0 +1,27 @@
+import { renderHook, waitFor } from '@testing-library/react';
+import mockRouter from 'next-router-mock';
+
+import { useFetchDomains } from '../../services/domains';
+import { createWrapper } from '../../__mocks__/utils';
+import { dummyDomain } from '../../__mocks__/data';
+
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+
+fetchMock.mockIf(`${window.location.origin}/api/domains`, async () => {
+   return new Promise((resolve) => {
+      resolve({
+         body: JSON.stringify({ domains: [dummyDomain] }),
+         status: 200,
+      });
+   });
+});
+
+describe('DomainHooks', () => {
+   it('useFetchDomains should fetch the Domains', async () => {
+      const { result } = renderHook(() => useFetchDomains(mockRouter), { wrapper: createWrapper() });
+      // const result = { current: { isSuccess: false, data: '' } };
+      await waitFor(() => {
+         return expect(result.current.isLoading).toBe(false);
+      });
+   });
+});

__tests__/pages/domain.test.tsx

@@ -1,17 +1,26 @@
 import { fireEvent, render, screen } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from 'react-query';
 import SingleDomain from '../../pages/domain/[slug]';
 import { useAddDomain, useDeleteDomain, useFetchDomains, useUpdateDomain } from '../../services/domains';
-import { useAddKeywords, useDeleteKeywords, useFavKeywords, useFetchKeywords, useRefreshKeywords } from '../../services/keywords';
-import { dummyDomain, dummyKeywords } from '../data';
+import { useAddKeywords, useDeleteKeywords,
+   useFavKeywords, useFetchKeywords, useRefreshKeywords, useFetchSingleKeyword } from '../../services/keywords';
+import { dummyDomain, dummyKeywords, dummySettings } from '../../__mocks__/data';
+import { useFetchSettings } from '../../services/settings';
 
 jest.mock('../../services/domains');
 jest.mock('../../services/keywords');
+jest.mock('../../services/settings');
+
 jest.mock('next/router', () => ({
    useRouter: () => ({
      query: { slug: dummyDomain.slug },
    }),
 }));
 
+jest.mock('react-chartjs-2', () => ({
+   Line: () => null,
+}));
+
 const useFetchDomainsFunc = useFetchDomains as jest.Mock<any>;
 const useFetchKeywordsFunc = useFetchKeywords as jest.Mock<any>;
 const useDeleteKeywordsFunc = useDeleteKeywords as jest.Mock<any>;
@@ -21,11 +30,17 @@ const useAddDomainFunc = useAddDomain as jest.Mock<any>;
 const useAddKeywordsFunc = useAddKeywords as jest.Mock<any>;
 const useUpdateDomainFunc = useUpdateDomain as jest.Mock<any>;
 const useDeleteDomainFunc = useDeleteDomain as jest.Mock<any>;
+const useFetchSettingsFunc = useFetchSettings as jest.Mock<any>;
+const useFetchSingleKeywordFunc = useFetchSingleKeyword as jest.Mock<any>;
 
 describe('SingleDomain Page', () => {
+   const queryClient = new QueryClient();
    beforeEach(() => {
+      useFetchSettingsFunc.mockImplementation(() => ({ data: { settings: dummySettings }, isLoading: false }));
       useFetchDomainsFunc.mockImplementation(() => ({ data: { domains: [dummyDomain] }, isLoading: false }));
       useFetchKeywordsFunc.mockImplementation(() => ({ keywordsData: { keywords: dummyKeywords }, keywordsLoading: false }));
+      const fetchPayload = { history: dummyKeywords[0].history || [], searchResult: dummyKeywords[0].lastResult || [] };
+      useFetchSingleKeywordFunc.mockImplementation(() => ({ data: fetchPayload, isLoading: false }));
       useDeleteKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
       useFavKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
       useRefreshKeywordsFunc.mockImplementation(() => ({ mutate: () => { } }));
@@ -38,158 +53,154 @@ describe('SingleDomain Page', () => {
       jest.clearAllMocks();
    });
    it('Render without crashing.', async () => {
-      const { getByTestId } = render(<SingleDomain />);
-      // screen.debug(undefined, Infinity);
-      expect(getByTestId('domain-header')).toBeInTheDocument();
-      // expect(await result.findByText(/compressimage/i)).toBeInTheDocument();
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      expect(screen.getByTestId('domain-header')).toBeInTheDocument();
    });
    it('Should Call the useFetchDomains hook on render.', async () => {
-      render(<SingleDomain />);
-      // screen.debug(undefined, Infinity);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       expect(useFetchDomains).toHaveBeenCalled();
-      // expect(await result.findByText(/compressimage/i)).toBeInTheDocument();
    });
    it('Should Render the Keywords', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(2);
    });
    it('Should Display the Keywords Details Sidebar on Keyword Click.', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const keywords = document.querySelectorAll('.keyword');
       const firstKeyword = keywords && keywords[0].querySelector('a');
-      if (firstKeyword) fireEvent(firstKeyword, new MouseEvent('click', { bubbles: true }));
+      if (firstKeyword) fireEvent.click(firstKeyword);
+      expect(useFetchSingleKeyword).toHaveBeenCalled();
       expect(screen.getByTestId('keywordDetails')).toBeVisible();
    });
    it('Should Display the AddDomain Modal on Add Domain Button Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=add_domain]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('add_domain');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('adddomain_modal')).toBeVisible();
    });
    it('Should Display the AddKeywords Modal on Add Keyword Button Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=add_keyword]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('add_keyword');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('addkeywords_modal')).toBeVisible();
    });
 
    it('Should display the Domain Settings on Settings Button click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=show_domain_settings]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('show_domain_settings');
+      if (button) fireEvent.click(button);
       expect(screen.getByTestId('domain_settings')).toBeVisible();
    });
 
    it('Device Tab change should be functioning.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=mobile_tab]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('mobile_tab');
+      if (button) fireEvent.click(button);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(0);
    });
 
    it('Search Filter should function properly', async () => {
-      render(<SingleDomain />);
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
       const inputNode = screen.getByTestId('filter_input');
-      fireEvent.change(inputNode, { target: { value: 'compressor' } }); // triggers onChange event
+      if (inputNode) fireEvent.change(inputNode, { target: { value: 'compressor' } }); // triggers onChange event
       expect(inputNode.getAttribute('value')).toBe('compressor');
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(1);
    });
 
    it('Country Filter should function properly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=filter_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('filter_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.country_filter')).toBeVisible();
 
       const countrySelect = document.querySelector('.country_filter .selected');
-      if (countrySelect) fireEvent(countrySelect, new MouseEvent('click', { bubbles: true }));
+      if (countrySelect) fireEvent.click(countrySelect);
       expect(document.querySelector('.country_filter .select_list')).toBeVisible();
       const firstCountry = document.querySelector('.country_filter .select_list ul li:nth-child(1)');
-      if (firstCountry) fireEvent(firstCountry, new MouseEvent('click', { bubbles: true }));
+      if (firstCountry) fireEvent.click(firstCountry);
       const keywordsCount = document.querySelectorAll('.keyword').length;
       expect(keywordsCount).toBe(0);
    });
 
    // Tags Filter should function properly
    it('Tags Filter should Render & Function properly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=filter_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('filter_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.tags_filter')).toBeVisible();
 
       const countrySelect = document.querySelector('.tags_filter .selected');
-      if (countrySelect) fireEvent(countrySelect, new MouseEvent('click', { bubbles: true }));
+      if (countrySelect) fireEvent.click(countrySelect);
       expect(document.querySelector('.tags_filter .select_list')).toBeVisible();
       expect(document.querySelectorAll('.tags_filter .select_list ul li').length).toBe(1);
 
       const firstTag = document.querySelector('.tags_filter .select_list ul li:nth-child(1)');
-      if (firstTag) fireEvent(firstTag, new MouseEvent('click', { bubbles: true }));
+      if (firstTag) fireEvent.click(firstTag);
       expect(document.querySelectorAll('.keyword').length).toBe(1);
    });
 
    it('Sort Options Should be visible Sort Button on Click.', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
       expect(document.querySelector('.sort_options')).toBeVisible();
    });
 
    it('Sort: Position should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
-
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(1)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(2)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });
 
    it('Sort: Date Added should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
 
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(3)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(4)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });
 
    it('Sort: Alphabetical should sort keywords accordingly', async () => {
-      render(<SingleDomain />);
-      const button = document.querySelector('[data-testid=sort_button]');
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      render(<QueryClientProvider client={queryClient}><SingleDomain /></QueryClientProvider>);
+      const button = screen.getByTestId('sort_button');
+      if (button) fireEvent.click(button);
 
       // Test Top Position Sort
       const topPosSortOption = document.querySelector('ul.sort_options li:nth-child(5)');
-      if (topPosSortOption) fireEvent(topPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (topPosSortOption) fireEvent.click(topPosSortOption);
       const firstKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(firstKeywordTitle).toBe('compress image');
 
       // Test Lowest Position Sort
-      if (button) fireEvent(button, new MouseEvent('click', { bubbles: true }));
+      if (button) fireEvent.click(button);
       const lowestPosSortOption = document.querySelector('ul.sort_options li:nth-child(6)');
-      if (lowestPosSortOption) fireEvent(lowestPosSortOption, new MouseEvent('click', { bubbles: true }));
+      if (lowestPosSortOption) fireEvent.click(lowestPosSortOption);
       const secondKeywordTitle = document.querySelector('.domKeywords_keywords .keyword:nth-child(1) a')?.textContent;
       expect(secondKeywordTitle).toBe('image compressor');
    });

__tests__/pages/domains.test.tsx

@@ -0,0 +1,49 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import { QueryClient, QueryClientProvider } from 'react-query';
+import * as ReactQuery from 'react-query';
+import { dummyDomain } from '../../__mocks__/data';
+import Domains from '../../pages/domains';
+
+jest.mock('next/router', () => jest.requireActual('next-router-mock'));
+jest.spyOn(ReactQuery, 'useQuery').mockImplementation(jest.fn().mockReturnValue(
+   { data: { domains: [dummyDomain] }, isLoading: false, isSuccess: true },
+));
+
+fetchMock.mockIf(`${window.location.origin}/api/domains`, async () => {
+   return new Promise((resolve) => {
+      resolve({
+         body: JSON.stringify({ domains: [dummyDomain] }),
+         status: 200,
+      });
+   });
+});
+
+describe('Domains Page', () => {
+   const queryClient = new QueryClient();
+   it('Renders without crashing', async () => {
+      render(
+          <QueryClientProvider client={queryClient}>
+              <Domains />
+          </QueryClientProvider>,
+      );
+      expect(screen.getByTestId('domains')).toBeInTheDocument();
+   });
+   it('Renders the Domain Component', async () => {
+      const { container } = render(
+          <QueryClientProvider client={queryClient}>
+              <Domains />
+          </QueryClientProvider>,
+      );
+      expect(container.querySelector('.domItem')).toBeInTheDocument();
+   });
+   it('Should Display Add Domain Modal on relveant Button Click.', async () => {
+      render(<QueryClientProvider client={queryClient}><Domains /></QueryClientProvider>);
+      const button = screen.getByTestId('addDomainButton');
+      if (button) fireEvent.click(button);
+      expect(screen.getByTestId('adddomain_modal')).toBeVisible();
+   });
+   it('Should Display the version number in Footer.', async () => {
+      render(<QueryClientProvider client={queryClient}><Domains /></QueryClientProvider>);
+      expect(screen.getByText('SerpBear v0.0.0')).toBeVisible();
+   });
+});

__tests__/pages/index.test.tsx

@@ -2,21 +2,16 @@ import { render, screen } from '@testing-library/react';
 import { QueryClient, QueryClientProvider } from 'react-query';
 import Home from '../../pages/index';
 
+const routerPush = jest.fn();
+jest.mock('next/router', () => ({
+   useRouter: () => ({
+      push: routerPush,
+   }),
+}));
+
 describe('Home Page', () => {
    const queryClient = new QueryClient();
    it('Renders without crashing', async () => {
-      // const dummyDomain = {
-      //    ID: 1,
-      //    domain: 'compressimage.io',
-      //    slug: 'compressimage-io',
-      //    keywordCount: 0,
-      //    lastUpdated: '2022-11-11T10:00:32.243',
-      //    added: '2022-11-11T10:00:32.244',
-      //    tags: [],
-      //    notification: true,
-      //    notification_interval: 'daily',
-      //    notification_emails: '',
-      // };
       render(
           <QueryClientProvider client={queryClient}>
               <Home />
@@ -26,12 +21,12 @@ describe('Home Page', () => {
       expect(await screen.findByRole('main')).toBeInTheDocument();
       expect(screen.queryByText('Add Domain')).not.toBeInTheDocument();
    });
-   it('Should Display the Add Domain Modal when there are no Domains.', async () => {
+   it('Should redirect to /domains route.', async () => {
        render(
            <QueryClientProvider client={queryClient}>
                <Home />
            </QueryClientProvider>,
        );
-       expect(await screen.findByText('Add Domain')).toBeInTheDocument();
+       expect(routerPush).toHaveBeenCalledWith('/domains');
    });
 });

components/common/Chart.tsx

@@ -6,19 +6,21 @@ ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, T
 
 type ChartProps ={
    labels: string[],
-   sreies: number[]
+   sreies: number[],
+   reverse? : boolean,
+   noMaxLimit?: boolean
 }
 
-const Chart = ({ labels, sreies }:ChartProps) => {
+const Chart = ({ labels, sreies, reverse = true, noMaxLimit = false }:ChartProps) => {
    const options = {
       responsive: true,
       maintainAspectRatio: false,
       animation: false as const,
       scales: {
          y: {
-            reverse: true,
+            reverse,
             min: 1,
-            max: 100,
+            max: !noMaxLimit && reverse ? 100 : undefined,
          },
       },
       plugins: {

components/common/ChartSlim.tsx

@@ -6,10 +6,12 @@ ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Filler,
 
 type ChartProps ={
    labels: string[],
-   sreies: number[]
+   sreies: number[],
+   noMaxLimit?: boolean,
+   reverse?: boolean
 }
 
-const ChartSlim = ({ labels, sreies }:ChartProps) => {
+const ChartSlim = ({ labels, sreies, noMaxLimit = false, reverse = true }:ChartProps) => {
    const options = {
       responsive: true,
       maintainAspectRatio: false,
@@ -17,9 +19,9 @@ const ChartSlim = ({ labels, sreies }:ChartProps) => {
       scales: {
          y: {
             display: false,
-            reverse: true,
+            reverse,
             min: 1,
-            max: 100,
+            max: noMaxLimit ? undefined : 100,
          },
          x: {
             display: false,
@@ -35,7 +37,7 @@ const ChartSlim = ({ labels, sreies }:ChartProps) => {
      },
    };
 
-   return <div className='w-[120px] h-[30px] rounded border border-gray-200'>
+   return <div className='w-[80px] h-[30px] rounded border border-gray-200'>
          <Line
             datasetIdKey='XXX'
             options={options}

components/common/Footer.tsx

@@ -0,0 +1,32 @@
+import { useState } from 'react';
+import { CSSTransition } from 'react-transition-group';
+import { useFetchChangelog } from '../../services/misc';
+import ChangeLog from '../settings/Changelog';
+
+interface FooterProps {
+   currentVersion: string
+}
+
+const Footer = ({ currentVersion = '' }: FooterProps) => {
+   const [showChangelog, setShowChangelog] = useState(false);
+   const { data: changeLogs } = useFetchChangelog();
+   const latestVersionNum = changeLogs && Array.isArray(changeLogs) && changeLogs[0] ? changeLogs[0].name : '';
+
+   return (
+      <footer className='text-center flex flex-1 justify-center pb-5 items-end'>
+         <span className='text-gray-500 text-xs'>
+            <a className='cursor-pointer' onClick={() => setShowChangelog(true)}>SerpBear v{currentVersion || '0.0.0'}</a>
+            {currentVersion && latestVersionNum && `v${currentVersion}` !== latestVersionNum && (
+               <a className='cursor-pointer text-indigo-700 font-semibold' onClick={() => setShowChangelog(true)}>
+                  {' '}| Update to Version {latestVersionNum} (latest)
+               </a>
+            )}
+         </span>
+         <CSSTransition in={showChangelog} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
+             <ChangeLog closeChangeLog={() => setShowChangelog(false)} />
+         </CSSTransition>
+      </footer>
+   );
+};
+
+export default Footer;

components/common/Icon.tsx

@@ -10,10 +10,10 @@ type IconProps = {
 }
 
 const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '' }: IconProps) => {
-   const xmlnsProps = { xmlns: 'http://www.w3.org/2000/svg', xmlnsXlink: 'http://www.w3.org/1999/xlink', preserveAspectRatio: 'xMidYMid meet' };
+   const xmlnsProps = { title, xmlns: 'http://www.w3.org/2000/svg', xmlnsXlink: 'http://www.w3.org/1999/xlink', preserveAspectRatio: 'xMidYMid meet' };
 
    return (
-       <span className={`icon inline-block relative top-[2px] ${classes}`} title={title}>
+       <span className={`icon inline-block relative top-[2px] ${classes}`}>
          {type === 'logo'
             && <svg {...xmlnsProps} width={size} viewBox="0 0 1484.32 1348.5">
                <path fill={color} d="M1406.23,604.17s-44-158.18,40.43-192.67,195,97.52,195,97.52,314-65.41,534,0c0,0,122.16-105.61,214.68-80.28,99.9,27.36,32.7,181.38,32.7,181.38s228.36,384.15,239.06,737.38c0,0-346.1,346.09-746.9,406.75,0,0-527.47-106.44-737.38-449.57C1177.88,1304.68,1169.55,1008.54,1406.23,604.17Z" transform="translate(-1177.84 -405.75)"/>
@@ -131,13 +131,13 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
          }
          {type === 'star'
             && <svg width={size} viewBox="0 0 24 24" {...xmlnsProps}>
-               <path fill={color} d="M10 1L7 7l-6 .75l4.13 4.62L4 19l6-3l6 3l-1.12-6.63L19 7.75L13 7zm0 2.24l2.34 4.69l4.65.58l-3.18 3.56l.87 5.15L10 14.88l-4.68 2.34l.87-5.15l-3.18-3.56l4.65-.58z"/>
-            </svg>
+                  <path fill={color} d="m12 15.39l-3.76 2.27l.99-4.28l-3.32-2.88l4.38-.37L12 6.09l1.71 4.04l4.38.37l-3.32 2.88l.99 4.28M22 9.24l-7.19-.61L12 2L9.19 8.63L2 9.24l5.45 4.73L5.82 21L12 17.27L18.18 21l-1.64-7.03z"></path>
+               </svg>
          }
          {type === 'star-filled'
             && <svg width={size} viewBox="0 0 24 24" {...xmlnsProps}>
-               <path fill={color} d="M10 1l3 6l6 .75l-4.12 4.62L16 19l-6-3l-6 3l1.13-6.63L1 7.75L7 7z"/>
-            </svg>
+                  <path fill={color} d="M12 17.27L18.18 21l-1.64-7.03L22 9.24l-7.19-.62L12 2L9.19 8.62L2 9.24l5.45 4.73L5.82 21z"></path>
+               </svg>
          }
          {type === 'link'
             && <svg width={size} viewBox="0 0 20 20" {...xmlnsProps}>
@@ -186,6 +186,133 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = '
                <path d="M5.5 5h13a1 1 0 0 1 .5 1.5L14 12v7l-4-3v-4L5 6.5A1 1 0 0 1 5.5 5" fill="none" stroke={color} strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"/>
             </svg>
          }
+         {type === 'idea'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <g fill="none" stroke={color} strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
+                     <path d="M3 12h1m8-9v1m8 8h1M5.6 5.6l.7.7m12.1-.7l-.7.7"/>
+                     <path d="M9 16a5 5 0 1 1 6 0a3.5 3.5 0 0 0-1 3a2 2 0 0 1-4 0a3.5 3.5 0 0 0-1-3"/>
+                     <path d="M9.7 17h4.6"/>
+                  </g>
+               </svg>
+         }
+         {type === 'tracking'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill={color} d="M21 7a.78.78 0 0 0 0-.21a.64.64 0 0 0-.05-.17a1.1 1.1 0 0 0-.09-.14a.75.75 0 0 0-.14-.17l-.12-.07a.69.69 0 0 0-.19-.1h-.2A.7.7 0 0 0 20 6h-5a1 1 0 0 0 0 2h2.83l-4 4.71l-4.32-2.57a1 1 0 0 0-1.28.22l-5 6a1 1 0 0 0 .13 1.41A1 1 0 0 0 4 18a1 1 0 0 0 .77-.36l4.45-5.34l4.27 2.56a1 1 0 0 0 1.27-.21L19 9.7V12a1 1 0 0 0 2 0V7z"/>
+               </svg>
+         }
+         {type === 'google'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 256 262">
+                  <path d="M255.878 133.451c0-10.734-.871-18.567-2.756-26.69H130.55v48.448h71.947c-1.45 12.04-9.283 30.172-26.69 42.356l-.244 1.622l38.755 30.023l2.685.268c24.659-22.774 38.875-56.282 38.875-96.027" fill="#4285F4"/>
+                  <path d="M130.55 261.1c35.248 0 64.839-11.605 86.453-31.622l-41.196-31.913c-11.024 7.688-25.82 13.055-45.257 13.055c-34.523 0-63.824-22.773-74.269-54.25l-1.531.13l-40.298 31.187l-.527 1.465C35.393 231.798 79.49 261.1 130.55 261.1" fill="#34A853"/>
+                  <path d="M56.281 156.37c-2.756-8.123-4.351-16.827-4.351-25.82c0-8.994 1.595-17.697 4.206-25.82l-.073-1.73L15.26 71.312l-1.335.635C5.077 89.644 0 109.517 0 130.55s5.077 40.905 13.925 58.602l42.356-32.782" fill="#FBBC05"/>
+                  <path d="M130.55 50.479c24.514 0 41.05 10.589 50.479 19.438l36.844-35.974C195.245 12.91 165.798 0 130.55 0C79.49 0 35.393 29.301 13.925 71.947l42.211 32.783c10.59-31.477 39.891-54.251 74.414-54.251" fill="#EB4335" />
+               </svg>
+         }
+         {type === 'adwords'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 256 256">
+                  <g>
+                     <path d="M5.888,166.405103 L90.88,20.9 C101.676138,27.2558621 156.115862,57.3844138 164.908138,63.1135172 L79.9161379,208.627448 C70.6206897,220.906621 -5.888,185.040138 5.888,166.396276 L5.888,166.405103 Z" fill="#FBBC04"></path>
+                     <path d="M250.084224,166.401789 L165.092224,20.9055131 C153.210293,1.13172 127.619121,-6.05393517 106.600638,5.62496138 C85.582155,17.3038579 79.182155,42.4624786 91.0640861,63.1190303 L176.056086,208.632961 C187.938017,228.397927 213.52919,235.583582 234.547672,223.904686 C254.648086,212.225789 261.966155,186.175582 250.084224,166.419444 L250.084224,166.401789 Z" fill="#4285F4"></path>
+                     <ellipse fill="#34A853" cx="42.6637241" cy="187.924414" rx="42.6637241" ry="41.6044138"></ellipse>
+                  </g>
+               </svg>
+         }
+         {type === 'keywords'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill="none" stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M5 8h14M5 12h14M5 16h6"></path>
+               </svg>
+         }
+         {type === 'integration'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill="none" stroke={color} strokeWidth={2} d="M10 21c-2.5 2.5-5 2-7 0s-2.5-4.5 0-7l3-3l7 7zm4-18c2.5-2.5 5-2 7.001 0c2.001 2 2.499 4.5 0 7l-3 3L11 6zm-3 7l-2.5 2.5zm3 3l-2.5 2.5z"></path>
+               </svg>
+         }
+
+         {type === 'cursor'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill="none" stroke={color} strokeWidth="2" d="M6 3l12 11l-5 1l3 5.5l-3 1.5l-3-6l-4 3z"/>
+               </svg>
+         }
+         {type === 'eye'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <g fill="none">
+                     <path d="M21.257 10.962c.474.62.474 1.457 0 2.076C19.764 14.987 16.182 19 12 19c-4.182 0-7.764-4.013-9.257-5.962a1.692 1.692 0 0 1 0-2.076C4.236 9.013 7.818 5 12 5c4.182 0 7.764 4.013 9.257 5.962z" stroke={color} strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"/>
+                     <circle cx="12" cy="12" r="3" stroke={color} strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"/>
+                  </g>
+               </svg>
+         }
+         {type === 'eye-closed'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <g fill="none" stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth="2">
+                     <path d="M6.873 17.129c-1.845-1.31-3.305-3.014-4.13-4.09a1.693 1.693 0 0 1 0-2.077C4.236 9.013 7.818 5 12 5c1.876 0 3.63.807 5.13 1.874"/>
+                     <path d="M14.13 9.887a3 3 0 1 0-4.243 4.242M4 20L20 4M10 18.704A7.124 7.124 0 0 0 12 19c4.182 0 7.764-4.013 9.257-5.962a1.694 1.694 0 0 0-.001-2.078A22.939 22.939 0 0 0 19.57 9"/>
+                  </g>
+               </svg>
+         }
+         {type === 'target'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path d="M19.938 13A8.004 8.004 0 0 1 13 19.938V22h-2v-2.062A8.004 8.004 0 0 1 4.062 13H2v-2h2.062A8.004 8.004 0 0 1 11 4.062V2h2v2.062A8.004 8.004 0 0 1 19.938 11H22v2h-2.062zM12 18a6 6 0 1 0 0-12a6 6 0 0 0 0 12zm0-3a3 3 0 1 0 0-6a3 3 0 0 0 0 6z" fill={color} fillRule="nonzero"/>
+               </svg>
+         }
+         {type === 'help'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path d="M12 4c4.411 0 8 3.589 8 8s-3.589 8-8 8s-8-3.589-8-8s3.589-8 8-8m0-2C6.477 2 2 6.477 2 12s4.477 10 10 10s10-4.477 10-10S17.523 2 12 2zm4 8a4 4 0 0 0-8 0h2c0-1.103.897-2 2-2s2 .897 2 2s-.897 2-2 2a1 1 0 0 0-1 1v2h2v-1.141A3.991 3.991 0 0 0 16 10zm-3 6h-2v2h2v-2z" fill={color} />
+               </svg>
+         }
+         {type === 'date'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path d="M22 2.25h-3.25V.75a.75.75 0 0 0-1.5-.001V2.25h-4.5V.75a.75.75 0 0 0-1.5-.001V2.25h-4.5V.75a.75.75 0 0 0-1.5-.001V2.25H2a2 2 0 0 0-2 1.999v17.75a2 2 0 0 0 2 2h20a2 2 0 0 0 2-2V4.249a2 2 0 0 0-2-1.999zM22.5 22a.5.5 0 0 1-.499.5H2a.5.5 0 0 1-.5-.5V4.25a.5.5 0 0 1 .5-.499h3.25v1.5a.75.75 0 0 0 1.5.001V3.751h4.5v1.5a.75.75 0 0 0 1.5.001V3.751h4.5v1.5a.75.75 0 0 0 1.5.001V3.751H22a.5.5 0 0 1 .499.499z" fill={color} />
+                  <path d="M5.25 9h3v2.25h-3z" fill={color} />
+                  <path d="M5.25 12.75h3V15h-3z" fill={color} />
+                  <path d="M5.25 16.5h3v2.25h-3z" fill={color} />
+                  <path d="M10.5 16.5h3v2.25h-3z" fill={color} />
+                  <path d="M10.5 12.75h3V15h-3z" fill={color} />
+                  <path d="M10.5 9h3v2.25h-3z" fill={color} />
+                  <path d="M15.75 16.5h3v2.25h-3z" fill={color} />
+                  <path d="M15.75 12.75h3V15h-3z" fill={color} />
+                  <path d="M15.75 9h3v2.25h-3z" fill={color} />
+               </svg>
+         }
+         {type === 'email'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill={color} d="M22 6c0-1.1-.9-2-2-2H4c-1.1 0-2 .9-2 2v12c0 1.1.9 2 2 2h16c1.1 0 2-.9 2-2zm-2 0l-8 5l-8-5zm0 12H4V8l8 5l8-5z" />
+               </svg>
+         }
+         {type === 'scraper'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 16 16">
+               <path fill={color} d="M1 3.5A2.5 2.5 0 0 1 3.5 1h7A2.5 2.5 0 0 1 13 3.5v1.53a4.538 4.538 0 0 0-1-.004V5H2v5.5A1.5 1.5 0 0 0 3.5 12h2.954l-.72.72a2.52 2.52 0 0 0-.242.28H3.5A2.5 2.5 0 0 1 1 10.5zm7.931 3.224l-.577-.578a.5.5 0 1 0-.708.708l.745.744c.144-.306.324-.6.54-.874M2 4h10v-.5A1.5 1.5 0 0 0 10.5 2h-7A1.5 1.5 0 0 0 2 3.5zm4.354 2.854a.5.5 0 1 0-.708-.708l-2 2a.5.5 0 0 0 0 .708l2 2a.5.5 0 0 0 .708-.708L4.707 8.5zm6.538-.83c.366.042.471.48.21.742l-.975.975a1.507 1.507 0 1 0 2.132 2.132l.975-.975c.261-.261.7-.156.742.21a3.518 3.518 0 0 1-4.676 3.723l-2.726 2.727a1.507 1.507 0 1 1-2.132-2.132L9.168 10.7a3.518 3.518 0 0 1 3.724-4.676" />
+            </svg>
+         }
+         {type === 'city'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 48 48">
+                  <g fill="none">
+                     <path stroke={color} strokeLinecap="round" strokeLinejoin="round" strokeWidth={4} d="M4 42h40"></path>
+                     <rect width={8} height={16} x={8} y={26} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path stroke={color} strokeLinecap="square" strokeLinejoin="round" strokeWidth={4} d="M12 34h1"></path>
+                     <rect width={24} height={38} x={16} y={4} stroke={color} strokeLinejoin="round" strokeWidth={4} rx={2}></rect>
+                     <path fill={color} d="M22 10h4v4h-4zm8 0h4v4h-4zm-8 7h4v4h-4zm8 0h4v4h-4zm0 7h4v4h-4zm0 7h4v4h-4z"></path>
+                  </g>
+               </svg>
+         }
+         {type === 'research'
+            && <svg width={size} viewBox="0 0 48 48" {...xmlnsProps}>
+                  <g fill="none" stroke={color} strokeWidth={4}>
+                     <path strokeLinecap="round" d="M4 7h40M4 23h11M4 39h11"></path>
+                     <path d="M31.5 34a8.5 8.5 0 1 0 0-17a8.5 8.5 0 0 0 0 17Z"></path>
+                     <path strokeLinecap="round" d="m37 32l7 7.05"></path>
+                  </g>
+               </svg>
+         }
+         {type === 'domains'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 56 56">
+                  <path fill={color} d="M7.328 43.504c.445 0 .914-.14 1.383-.469V17.957c0-.844.164-1.172.914-1.57L31.352 3.87c.07-1.547-.915-2.508-2.25-2.508c-.61 0-1.266.164-1.97.586L7.493 13.246c-2.297 1.336-2.578 1.758-2.578 4.43v22.43c0 2.015.96 3.398 2.414 3.398m9.375 5.414c.422 0 .89-.14 1.383-.469V23.371c0-.914.117-1.148.89-1.57L40.703 9.26c.07-1.523-.89-2.507-2.25-2.507c-.586 0-1.266.187-1.945.562L16.82 18.636c-2.297 1.313-2.555 1.805-2.555 4.43V45.52c0 2.015 1.008 3.398 2.438 3.398m10.031 5.719c.82 0 1.805-.328 2.977-.985l18.375-10.547c2.156-1.242 3-2.53 3-5.156l-.047-21.234c0-2.813-1.008-4.242-2.766-4.242c-.773 0-1.758.304-2.859.937L26.992 24.027c-2.203 1.29-2.977 2.602-2.977 5.157v21.234c0 2.719.961 4.219 2.72 4.219M28 50.067c-.117-.024-.164-.094-.164-.258L28 29.254c0-.89.258-1.36 1.055-1.805l17.742-10.43c.07-.046.14-.046.234-.023c.094.024.164.094.164.258l-.07 20.625c0 .773-.281 1.36-1.055 1.828L28.234 50.043a.284.284 0 0 1-.234.023"></path>
+               </svg>
+         }
+         {type === 'lock'
+            && <svg {...xmlnsProps} width={size} viewBox="0 0 24 24">
+                  <path fill={color} d="M6 22q-.825 0-1.412-.587T4 20V10q0-.825.588-1.412T6 8h1V6q0-2.075 1.463-3.537T12 1t3.538 1.463T17 6v2h1q.825 0 1.413.588T20 10v10q0 .825-.587 1.413T18 22zm6-5q.825 0 1.413-.587T14 15t-.587-1.412T12 13t-1.412.588T10 15t.588 1.413T12 17M9 8h6V6q0-1.25-.875-2.125T12 3t-2.125.875T9 6z" />
+               </svg>
+         }
        </span>
    );
  };

components/common/InputField.tsx

@@ -0,0 +1,28 @@
+type InputFieldProps = {
+   label: string;
+   value: string;
+   onChange: Function;
+   placeholder?: string;
+   classNames?: string;
+   hasError?: boolean;
+}
+
+const InputField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: InputFieldProps) => {
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+   return (
+      <div className="field--input w-full relative flex justify-between items-center">
+         <label className={labelStyle}>{label}</label>
+         <input
+            className={`p-2 border border-gray-200 rounded focus:outline-none w-[210px]
+             focus:border-blue-200 ${hasError ? ' border-red-400 focus:border-red-400' : ''} `}
+            type={'text'}
+            value={value}
+            onChange={(event) => onChange(event.target.value)}
+            autoComplete="off"
+            placeholder={placeholder}
+         />
+      </div>
+   );
+};
+
+export default InputField;

components/common/Modal.tsx

@@ -1,25 +1,17 @@
-import React, { useEffect } from 'react';
+import React from 'react';
 import Icon from './Icon';
+import useOnKey from '../../hooks/useOnKey';
 
 type ModalProps = {
    children: React.ReactNode,
    width?: string,
    title?: string,
+   verticalCenter?: boolean,
    closeModal: Function,
 }
 
-const Modal = ({ children, width = '1/2', closeModal, title }:ModalProps) => {
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            closeModal();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeModal]);
+const Modal = ({ children, width = '1/2', closeModal, title, verticalCenter = false }:ModalProps) => {
+   useOnKey('Escape', closeModal);
 
    const closeOnBGClick = (e:React.SyntheticEvent) => {
       e.stopPropagation();
@@ -30,8 +22,9 @@ const Modal = ({ children, width = '1/2', closeModal, title }:ModalProps) => {
    return (
       <div className='modal fixed top-0 left-0 bg-white/[.7] w-full h-screen z-50' onClick={closeOnBGClick}>
          <div
-         className={`modal__content max-w-[340px] absolute top-1/4 left-0 right-0 ml-auto mr-auto w-${width} 
-         lg:max-w-md bg-white shadow-md rounded-md p-5 border-t-[1px] border-gray-100 text-base`}>
+         className={`modal__content max-w-[340px] absolute left-0 right-0 ml-auto mr-auto w-${width} 
+         lg:max-w-md bg-white shadow-md rounded-md p-5 border-t-[1px] border-gray-100 text-base 
+         ${verticalCenter ? ' top-1/2 translate-y-[-50%]' : 'top-1/4'}`}>
             {title && <h3 className=' font-semibold mb-3'>{title}</h3>}
             <button
             className='modal-close absolute right-2 top-2 p-2 cursor-pointer text-gray-400 transition-all

components/common/SecretField.tsx

@@ -0,0 +1,37 @@
+import { useState } from 'react';
+import Icon from './Icon';
+
+type SecretFieldProps = {
+   label: string;
+   value: string;
+   onChange: Function;
+   placeholder?: string;
+   classNames?: string;
+   hasError?: boolean;
+}
+
+const SecretField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: SecretFieldProps) => {
+   const [showValue, setShowValue] = useState(false);
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+   return (
+      <div className="settings__section__secret w-full relative flex justify-between items-center">
+         <label className={labelStyle}>{label}</label>
+         <span
+         className="absolute top-1 right-0 px-2 py-1 cursor-pointer text-gray-400 select-none"
+         onClick={() => setShowValue(!showValue)}>
+            <Icon type={showValue ? 'eye-closed' : 'eye'} size={18} />
+         </span>
+         <input
+            className={`w-[210px] p-2 border border-gray-200 rounded focus:outline-none 
+             focus:border-blue-200 ${hasError ? ' border-red-400 focus:border-red-400' : ''} `}
+            type={showValue ? 'text' : 'password'}
+            value={value}
+            onChange={(event) => onChange(event.target.value)}
+            autoComplete="off"
+            placeholder={placeholder}
+         />
+      </div>
+   );
+};
+
+export default SecretField;

components/common/SelectField.tsx

@@ -9,12 +9,15 @@ type SelectFieldProps = {
    defaultLabel: string,
    options: SelectionOption[],
    selected: string[],
+   label?: string,
    multiple?: boolean,
    updateField: Function,
+   fullWidth?: boolean,
    minWidth?: number,
    maxHeight?: number|string,
    rounded?: string,
    flags?: boolean,
+   inline?: boolean,
    emptyMsg?: string
 }
 const SelectField = (props: SelectFieldProps) => {
@@ -26,8 +29,11 @@ const SelectField = (props: SelectFieldProps) => {
       updateField,
       minWidth = 180,
       maxHeight = 96,
+      fullWidth = false,
       rounded = 'rounded-3xl',
+      inline = false,
       flags = false,
+      label = '',
       emptyMsg = '' } = props;
 
    const [showOptions, setShowOptions] = useState<boolean>(false);
@@ -66,12 +72,13 @@ const SelectField = (props: SelectFieldProps) => {
    };
 
    return (
-       <div className="select font-semibold text-gray-500">
+       <div className={`select font-semibold text-gray-500 relative ${inline ? 'inline-block' : 'flex'} justify-between items-center"`}>
+         {label && <label className='mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'>{label}</label>}
          <div
-         className={`selected flex border ${rounded} p-1.5 px-4 cursor-pointer select-none w-[180px] min-w-[${minWidth}px] 
-         ${showOptions ? 'border-indigo-200' : ''}`}
+         className={`selected flex border ${rounded} p-1.5 px-4 cursor-pointer select-none ${fullWidth ? 'w-full' : 'w-[210px]'} 
+         min-w-[${minWidth}px] ${showOptions ? 'border-indigo-200' : ''}`}
          onClick={() => setShowOptions(!showOptions)}>
-            <span className={`w-[${minWidth - 30}px] inline-block truncate mr-2`}>
+            <span className={'w-full inline-block truncate mr-2 capitalize'}>
                {selected.length > 0 ? (selectedLabels.slice(0, 2).join(', ')) : defaultLabel}
             </span>
             {multiple && selected.length > 2
@@ -80,7 +87,7 @@ const SelectField = (props: SelectFieldProps) => {
          </div>
          {showOptions && (
             <div
-            className={`select_list mt-1 border absolute min-w-[${minWidth}px] 
+            className={`select_list mt-1 border absolute min-w-[${minWidth}px] top-[30px] right-0 ${fullWidth ? 'w-full' : 'w-[210px]'}
             ${rounded === 'rounded-3xl' ? 'rounded-lg' : rounded} max-h-${maxHeight} bg-white z-50 overflow-y-auto styled-scrollbar`}>
                {options.length > 20 && (
                   <div className=''>
@@ -99,7 +106,7 @@ const SelectField = (props: SelectFieldProps) => {
                      return (
                         <li
                         key={opt.value}
-                        className={`select-none cursor-pointer px-3 py-2 hover:bg-[#FCFCFF] 
+                        className={`select-none cursor-pointer px-3 py-2 hover:bg-[#FCFCFF] capitalize text-ellipsis overflow-hidden
                         ${itemActive ? ' bg-indigo-50 text-indigo-600 hover:bg-indigo-50' : ''} `}
                         onClick={() => selectItem(opt)}
                         >

components/common/SidePanel.tsx

@@ -0,0 +1,37 @@
+import React, { useEffect, useState } from 'react';
+import Icon from './Icon';
+import useOnKey from '../../hooks/useOnKey';
+
+type SidePanelProps = {
+   children: React.ReactNode,
+   closePanel: Function,
+   title?: string,
+   width?: 'large' | 'medium' | 'small',
+   position?: 'left' | 'right'
+}
+const SidePanel = ({ children, closePanel, width, position = 'right', title = '' }:SidePanelProps) => {
+   useOnKey('Escape', closePanel);
+   const closeOnBGClick = (e:React.SyntheticEvent) => {
+      e.stopPropagation();
+      e.nativeEvent.stopImmediatePropagation();
+      if (e.target === e.currentTarget) { closePanel(); }
+   };
+   return (
+       <div className="SidePanel fixed w-full h-screen top-0 left-0 z-50" onClick={closeOnBGClick}>
+         <div className={`absolute w-full max-w-md  border-l border-l-gray-400 bg-white customShadow top-0 
+         ${position === 'left' ? 'left-0' : 'right-0'} h-screen`}>
+            <div className='SidePanel__header px-5 py-4 text-slate-500 border-b border-b-gray-100'>
+               <h3 className=' text-black text-lg font-bold'>{title}</h3>
+               <button
+               className=' absolute top-2 right-2 p-2 px- text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
+               onClick={() => closePanel()}>
+                  <Icon type='close' size={24} />
+               </button>
+            </div>
+            <div>{children}</div>
+         </div>
+       </div>
+   );
+};
+
+export default SidePanel;

components/common/Sidebar.tsx

@@ -1,10 +1,11 @@
+/* eslint-disable @next/next/no-img-element */
 import React from 'react';
 import Link from 'next/link';
 import { useRouter } from 'next/router';
 import Icon from './Icon';
 
 type SidebarProps = {
-   domains: Domain[],
+   domains: DomainType[],
    showAddModal: Function
 }
 
@@ -23,13 +24,14 @@ const Sidebar = ({ domains, showAddModal } : SidebarProps) => {
                                  className={'my-2.5 leading-10'}>
                                     <Link href={`/domain/${d.slug}`} passHref={true}>
                                        <a className={`block cursor-pointer px-4 text-ellipsis max-w-[215px] overflow-hidden whitespace-nowrap rounded
-                                        rounded-r-none 
-                                        ${(`/domain/${d.slug}` === router.asPath ? 'bg-white text-zinc-800 border border-r-0' : 'text-zinc-500')}`}>
-                                          <i className={'text-center leading-4 mr-2 inline-block rounded-full w-5 h-5 bg-orange-200 not-italic'}>
-                                             {d.domain.charAt(0)}
-                                          </i>
+                                        rounded-r-none ${((`/domain/${d.slug}` === router.asPath || `/domain/console/${d.slug}` === router.asPath
+                                        || `/domain/insight/${d.slug}` === router.asPath || `/domain/ideas/${d.slug}` === router.asPath)
+                                        ? 'bg-white text-zinc-800 border border-r-0' : 'text-zinc-500')}`}>
+                                          <img
+                                          className={' inline-block mr-1'}
+                                          src={`https://www.google.com/s2/favicons?domain=${d.domain}&sz=16`} alt={d.domain}
+                                          />
                                           {d.domain}
-                                          {/* <span>0</span> */}
                                        </a>
                                     </Link>
                                  </li>)

components/common/ToggleField.tsx

@@ -0,0 +1,32 @@
+type ToggleFieldProps = {
+   label: string;
+   value: boolean;
+   onChange: (bool:boolean) => void ;
+   classNames?: string;
+}
+
+const ToggleField = ({ label = '', value = false, onChange, classNames = '' }: ToggleFieldProps) => {
+   return (
+      <div className={`field--toggle w-full relative ${classNames}`}>
+         <label className="relative inline-flex items-center cursor-pointer w-full justify-between">
+            <span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-auto">{label}</span>
+            <input
+            type="checkbox"
+            value={value.toString()}
+            checked={!!value}
+            className="sr-only peer"
+            onChange={() => onChange(!value)}
+            />
+            <div className="relative rounded-3xl w-9 h-5 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
+            peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
+            peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
+            after:absolute after:top-[2px] after:left-[2px] after:bg-white  after:border-gray-300
+            after:border after:rounded-full after:h-4 after:w-4
+            after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
+
+         </label>
+      </div>
+   );
+};
+
+export default ToggleField;

components/common/TopBar.tsx

@@ -1,3 +1,4 @@
+import Link from 'next/link';
 import { useRouter } from 'next/router';
 import React, { useState } from 'react';
 import toast from 'react-hot-toast';
@@ -11,6 +12,7 @@ type TopbarProps = {
 const TopBar = ({ showSettings, showAddModal }:TopbarProps) => {
    const [showMobileMenu, setShowMobileMenu] = useState<boolean>(false);
    const router = useRouter();
+   const isDomainsPage = router.pathname === '/domains';
 
    const logoutUser = async () => {
       try {
@@ -28,12 +30,21 @@ const TopBar = ({ showSettings, showAddModal }:TopbarProps) => {
    };
 
    return (
-       <div className="topbar flex w-full max-w-7xl mx-auto justify-between lg:justify-end bg-white lg:bg-transparent">
+       <div className={`topbar flex w-full mx-auto justify-between 
+       ${isDomainsPage ? 'max-w-5xl lg:justify-between' : 'max-w-7xl lg:justify-end'}  bg-white lg:bg-transparent`}>
 
-         <h3 className="p-4 text-base font-bold text-blue-700 lg:hidden">
+         <h3 className={`p-4 text-base font-bold text-blue-700 ${isDomainsPage ? 'lg:pl-0' : 'lg:hidden'}`}>
             <span className=' relative top-[3px] mr-1'><Icon type="logo" size={24} color="#364AFF" /></span> SerpBear
             <button className='px-3 py-1 font-bold text-blue-700  lg:hidden ml-3 text-lg' onClick={() => showAddModal()}>+</button>
          </h3>
+         {!isDomainsPage && router.asPath !== '/research' && (
+            <Link href={'/domains'} passHref={true}>
+               <a className=' right-14 top-2 px-2 py-1 cursor-pointer bg-[#ecf2ff] hover:bg-indigo-100 transition-all
+               absolute lg:top-3 lg:right-auto lg:left-8 lg:px-3 lg:py-2 rounded-full'>
+                  <Icon type="caret-left" size={16} title="Go Back" />
+               </a>
+            </Link>
+         )}
          <div className="topbar__right">
             <button className={' lg:hidden p-3'} onClick={() => setShowMobileMenu(!showMobileMenu)}>
                <Icon type="hamburger" size={24} />
@@ -41,16 +52,30 @@ const TopBar = ({ showSettings, showAddModal }:TopbarProps) => {
             <ul
             className={`text-sm font-semibold text-gray-500 absolute mt-[-10px] right-3 bg-white 
             border border-gray-200 lg:mt-2 lg:relative lg:block lg:border-0 lg:bg-transparent ${showMobileMenu ? 'block' : 'hidden'}`}>
-               <li className='block lg:inline-block lg:ml-5'>
-                  <a className='block px-3 py-2 cursor-pointer' href='https://docs.serpbear.com/' target="_blank" rel='noreferrer'>
-                     <Icon type="question" color={'#888'} size={14} /> Help
-                  </a>
+               <li className={`block lg:inline-block lg:ml-5 ${router.asPath === '/domains' ? ' text-blue-700' : ''}`}>
+                  <Link href={'/domains'} passHref={true}>
+                     <a className='block px-3 py-2 cursor-pointer'>
+                        <Icon type="domains" color={router.asPath === '/domains' ? '#1d4ed8' : '#888'} size={14} /> Domains
+                     </a>
+                  </Link>
+               </li>
+               <li className={`block lg:inline-block lg:ml-5 ${router.asPath === '/research' ? ' text-blue-700' : ''}`}>
+                  <Link href={'/research'} passHref={true}>
+                     <a className='block px-3 py-2 cursor-pointer'>
+                        <Icon type="research" color={router.asPath === '/research' ? '#1d4ed8' : '#888'} size={14} /> Research
+                     </a>
+                  </Link>
                </li>
                <li className='block lg:inline-block lg:ml-5'>
                   <a className='block px-3 py-2 cursor-pointer' onClick={() => showSettings()}>
                      <Icon type="settings-alt" color={'#888'} size={14} /> Settings
                   </a>
                </li>
+               <li className='block lg:inline-block lg:ml-5'>
+                  <a className='block px-3 py-2 cursor-pointer' href='https://docs.serpbear.com/' target="_blank" rel='noreferrer'>
+                     <Icon type="question" color={'#888'} size={14} /> Help
+                  </a>
+               </li>
                <li className='block lg:inline-block lg:ml-5'>
                   <a className='block px-3 py-2 cursor-pointer' onClick={() => logoutUser()}>
                      <Icon type="logout" color={'#888'} size={14} /> Logout

components/domains/AddDomain.tsx

@@ -1,53 +1,66 @@
 import React, { useState } from 'react';
 import Modal from '../common/Modal';
 import { useAddDomain } from '../../services/domains';
+import { isValidUrl } from '../../utils/client/validators';
 
 type AddDomainProps = {
+   domains: DomainType[],
    closeModal: Function
 }
 
-const AddDomain = ({ closeModal }: AddDomainProps) => {
+const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
    const [newDomain, setNewDomain] = useState<string>('');
-   const [newDomainError, setNewDomainError] = useState<boolean>(false);
+   const [newDomainError, setNewDomainError] = useState('');
    const { mutate: addMutate, isLoading: isAdding } = useAddDomain(() => closeModal());
 
    const addDomain = () => {
-      // console.log('ADD NEW DOMAIN', newDomain);
-      if (/^[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9](?:\.[a-zA-Z]{2,})+$/.test(newDomain.trim())) {
-         setNewDomainError(false);
-         // TODO: Domain Action
-         addMutate(newDomain.trim());
-      } else {
-         setNewDomainError(true);
+      setNewDomainError('');
+      const existingDomains = domains.map((d) => d.domain);
+      const insertedURLs = newDomain.split('\n');
+      const domainsTobeAdded:string[] = [];
+      const invalidDomains:string[] = [];
+      insertedURLs.forEach((url) => {
+        const theURL = url.trim();
+        if (isValidUrl(theURL)) {
+         const domURL = new URL(theURL);
+         const isDomain = domURL.pathname === '/';
+         if (isDomain && !existingDomains.includes(domURL.host)) {
+            domainsTobeAdded.push(domURL.host);
+         }
+         if (!isDomain && !existingDomains.includes(domURL.href)) {
+            const cleanedURL = domURL.href.replace('https://', '').replace('http://', '').replace(/^\/+|\/+$/g, '');
+            domainsTobeAdded.push(cleanedURL);
+         }
+        } else {
+         invalidDomains.push(theURL);
+        }
+      });
+      if (invalidDomains.length > 0) {
+         setNewDomainError(`Please Insert Valid Website URL. ${invalidDomains.length > 1 ? `Invalid URLs: ${invalidDomains.join(', ')}` : ''}`);
+      } else if (domainsTobeAdded.length > 0) {
+            console.log('domainsTobeAdded :', domainsTobeAdded);
+         addMutate(domainsTobeAdded);
       }
    };
 
-   const handleDomainInput = (e:React.FormEvent<HTMLInputElement>) => {
-      if (e.currentTarget.value === '' && newDomainError) { setNewDomainError(false); }
+   const handleDomainInput = (e:React.ChangeEvent<HTMLTextAreaElement>) => {
+      if (e.currentTarget.value === '' && newDomainError) { setNewDomainError(''); }
       setNewDomain(e.currentTarget.value);
    };
 
    return (
       <Modal closeModal={() => { closeModal(false); }} title={'Add New Domain'}>
          <div data-testid="adddomain_modal">
-            <h4 className='text-sm mt-4'>
-               Domain Name {newDomainError && <span className=' ml-2 block float-right text-red-500 text-xs font-semibold'>Not a Valid Domain</span>}
-            </h4>
-            <input
-               className={`w-full p-2 border border-gray-200 rounded mt-2 mb-3 focus:outline-none  focus:border-blue-200 
-               ${newDomainError ? ' border-red-400 focus:border-red-400' : ''} `}
-               type="text"
+            <h4 className='text-sm mt-4 pb-2'>Website URL(s)</h4>
+            <textarea
+               className={`w-full h-40 border rounded border-gray-200 p-4 outline-none
+                focus:border-indigo-300 ${newDomainError ? ' border-red-400 focus:border-red-400' : ''}`}
+               placeholder={'Type or Paste URLs here. Insert Each URL in a New line. eg: \nhttps://mysite.com/ \nhttps://anothersite.com/ '}
                value={newDomain}
-               placeholder={'example.com'}
-               onChange={handleDomainInput}
                autoFocus={true}
-               onKeyDown={(e) => {
-                  if (e.code === 'Enter') {
-                    e.preventDefault();
-                    addDomain();
-                  }
-               }}
-            />
+               onChange={handleDomainInput}>
+            </textarea>
+            {newDomainError && <div><span className=' ml-2 block float-right text-red-500 text-xs font-semibold'>{newDomainError}</span></div>}
             <div className='mt-6 text-right text-sm font-semibold'>
                <button className='py-2 px-5 rounded cursor-pointer bg-indigo-50 text-slate-500 mr-3' onClick={() => closeModal(false)}>Cancel</button>
                <button className='py-2 px-5 rounded cursor-pointer bg-blue-700 text-white' onClick={() => !isAdding && addDomain() }>

components/domains/DomainHeader.tsx

@@ -1,32 +1,44 @@
 import { useRouter } from 'next/router';
 import { useState } from 'react';
+import Link from 'next/link';
 import { useRefreshKeywords } from '../../services/keywords';
 import Icon from '../common/Icon';
 import SelectField from '../common/SelectField';
 
 type DomainHeaderProps = {
-   domain: Domain,
-   domains: Domain[],
+   domain: DomainType,
+   domains: DomainType[],
    showAddModal: Function,
    showSettingsModal: Function,
-   exportCsv:Function
+   exportCsv:Function,
+   scFilter?: string
+   setScFilter?: Function
+   showIdeaUpdateModal?:Function
 }
 
-const DomainHeader = ({ domain, showAddModal, showSettingsModal, exportCsv, domains }: DomainHeaderProps) => {
+const DomainHeader = (
+   { domain, showAddModal, showSettingsModal, exportCsv, domains, scFilter = 'thirtyDays', setScFilter, showIdeaUpdateModal }: DomainHeaderProps,
+) => {
    const router = useRouter();
    const [showOptions, setShowOptions] = useState<boolean>(false);
-
+   const [ShowSCDates, setShowSCDates] = useState<boolean>(false);
    const { mutate: refreshMutate } = useRefreshKeywords(() => {});
+   const isConsole = router.pathname === '/domain/console/[slug]';
+   const isInsight = router.pathname === '/domain/insight/[slug]';
+   const isIdeas = router.pathname === '/domain/ideas/[slug]';
 
+   const daysName = (dayKey:string) => dayKey.replace('three', '3').replace('seven', '7').replace('thirty', '30').replace('Days', ' Days');
    const buttonStyle = 'leading-6 inline-block px-2 py-2 text-gray-500 hover:text-gray-700';
    const buttonLabelStyle = 'ml-2 text-sm not-italic lg:invisible lg:opacity-0';
+   const tabStyle = 'rounded rounded-b-none cursor-pointer border-[#e9ebff] border-b-0';
+   const scDataFilterStlye = 'px-3 py-2 block w-full';
    return (
-      <div className='domain_kewywords_head flex w-full justify-between'>
+      <div className='domain_kewywords_head w-full '>
          <div>
             <h1 className="hidden lg:block text-xl font-bold my-3" data-testid="domain-header">
                {domain && domain.domain && <><i className=' capitalize font-bold not-italic'>{domain.domain.charAt(0)}</i>{domain.domain.slice(1)}</>}
             </h1>
-            <div className='bg-white mt-2 lg:hidden'>
+            <div className='domain_selector bg-white mt-2 lg:hidden'>
                <SelectField
                options={domains && domains.length > 0 ? domains.map((d) => { return { label: d.domain, value: d.slug }; }) : []}
                selected={[domain.slug]}
@@ -37,43 +49,125 @@ const DomainHeader = ({ domain, showAddModal, showSettingsModal, exportCsv, doma
                />
             </div>
          </div>
-         <div className='flex my-3'>
-            <button className={`${buttonStyle} lg:hidden`} onClick={() => setShowOptions(!showOptions)}>
+      <div className='flex w-full justify-between mt-4 lg:mt-0'>
+         <ul className=' max-w-[270px] overflow-auto flex items-end text-sm relative top-[2px] lg:max-w-none'>
+            <li className={`${tabStyle} ${router.pathname === '/domain/[slug]' ? 'bg-white border border-b-0 font-semibold' : ''}`}>
+               <Link href={`/domain/${domain.slug}`} passHref={true}>
+                  <a className='px-4 py-2 inline-block'><Icon type="tracking" color='#999' classes='hidden lg:inline-block' />
+                     <span className='text-xs lg:text-sm lg:ml-2'>Tracking</span>
+                  </a>
+               </Link>
+            </li>
+            <li className={`${tabStyle} ${router.pathname === '/domain/console/[slug]' ? 'bg-white border border-b-0 font-semibold' : ''}`}>
+               <Link href={`/domain/console/${domain.slug}`} passHref={true}>
+                  <a className='px-4 py-2 inline-block'><Icon type="google" size={13} classes='hidden lg:inline-block' />
+                     <span className='text-xs lg:text-sm lg:ml-2'>Discover</span>
+                     <Icon type='help' size={14} color="#aaa" classes="ml-2 hidden lg:inline-block" title='Discover Keywords you already Rank For' />
+                  </a>
+               </Link>
+            </li>
+            <li className={`${tabStyle} ${router.pathname === '/domain/insight/[slug]' ? 'bg-white border border-b-0 font-semibold' : ''}`}>
+               <Link href={`/domain/insight/${domain.slug}`} passHref={true}>
+                  <a className='px-4 py-2 inline-block'><Icon type="google" size={13} classes='hidden lg:inline-block' />
+                     <span className='text-xs lg:text-sm lg:ml-2'>Insight</span>
+                     <Icon type='help' size={14} color="#aaa" classes="ml-2 hidden lg:inline-block" title='Insight for Google Search Console Data' />
+                  </a>
+               </Link>
+            </li>
+            <li className={`${tabStyle} ${router.pathname === '/domain/ideas/[slug]' ? 'bg-white border border-b-0 font-semibold' : ''}`}>
+               <Link href={`/domain/ideas/${domain.slug}`} passHref={true}>
+                  <a className='px-4 py-2 inline-block'><Icon type="adwords" size={13} classes='hidden lg:inline-block' />
+                     <span className='text-xs lg:text-sm lg:ml-2'>Ideas</span>
+                     <Icon
+                     type='help'
+                     size={14}
+                     color="#aaa"
+                     classes="ml-2 hidden lg:inline-block"
+                     title='Get Keyword Ideas for this domain from Google Ads'
+                     />
+                  </a>
+               </Link>
+            </li>
+         </ul>
+         <div className={'flex mb-0 lg:mb-1 lg:mt-3'}>
+            {!isInsight && <button className={`${buttonStyle} lg:hidden`} onClick={() => setShowOptions(!showOptions)}>
                <Icon type='dots' size={20} />
             </button>
+            }
+            {isInsight && <button className={`${buttonStyle} lg:hidden invisible`}>x</button>}
             <div
             className={`hidden w-40 ml-[-70px] lg:block absolute mt-10 bg-white border border-gray-100 z-40 rounded 
             lg:z-auto lg:relative lg:mt-0 lg:border-0 lg:w-auto lg:bg-transparent`}
             style={{ display: showOptions ? 'block' : undefined }}>
-               <button
-               className={`domheader_action_button relative ${buttonStyle}`}
-               aria-pressed="false"
-               onClick={() => exportCsv()}>
-                  <Icon type='download' size={20} /><i className={`${buttonLabelStyle}`}>Export as csv</i>
-               </button>
-               <button
-               className={`domheader_action_button relative ${buttonStyle} lg:ml-3`}
-               aria-pressed="false"
-               onClick={() => refreshMutate({ ids: [], domain: domain.domain })}>
-                  <Icon type='reload' size={14} /><i className={`${buttonLabelStyle}`}>Reload All Serps</i>
-               </button>
+               {!isInsight && (
+                  <button
+                  className={`domheader_action_button relative ${buttonStyle}`}
+                  aria-pressed="false"
+                  onClick={() => exportCsv()}>
+                     <Icon type='download' size={20} /><i className={`${buttonLabelStyle}`}>Export as csv</i>
+                  </button>
+               )}
+               {!isConsole && !isInsight && !isIdeas && (
+                  <button
+                  className={`domheader_action_button relative ${buttonStyle} lg:ml-3`}
+                  aria-pressed="false"
+                  onClick={() => refreshMutate({ ids: [], domain: domain.domain })}>
+                     <Icon type='reload' size={14} /><i className={`${buttonLabelStyle}`}>Reload All Serps</i>
+                  </button>
+                )}
                <button
                data-testid="show_domain_settings"
                className={`domheader_action_button relative ${buttonStyle} lg:ml-3`}
                aria-pressed="false"
                onClick={() => showSettingsModal(true)}><Icon type='settings' size={20} />
-               <i className={`${buttonLabelStyle}`}>Domain Settings</i></button>
+                  <i className={`${buttonLabelStyle}`}>Domain Settings</i>
+               </button>
             </div>
-            <button
-            data-testid="add_keyword"
-            className={'ml-2 inline-block px-4 py-2 text-blue-700 font-bold text-sm'}
-            onClick={() => showAddModal(true)}>
-               <span
-               className='text-center leading-4 mr-2 inline-block rounded-full w-7 h-7 pt-1 bg-blue-700 text-white font-bold text-lg'>+</span>
-               <i className=' not-italic hidden lg:inline-block'>Add Keyword</i>
-            </button>
+            {!isConsole && !isInsight && !isIdeas && (
+               <button
+               data-testid="add_keyword"
+               className={'ml-2 inline-block text-blue-700 font-bold text-sm lg:px-4 lg:py-2'}
+               onClick={() => showAddModal(true)}>
+                  <span
+                  className='text-center leading-4 mr-2 inline-block rounded-full w-7 h-7 pt-1 bg-blue-700 text-white font-bold text-lg'>+</span>
+                  <i className=' not-italic hidden lg:inline-block'>Add Keyword</i>
+               </button>
+            )}
+            {isConsole && (
+               <div className='text-xs pl-4 ml-2 border-l border-gray-200 relative'>
+                  {/* <span className='hidden lg:inline-block'>Data From Last: </span> */}
+                  <span className='block cursor-pointer py-3' onClick={() => setShowSCDates(!ShowSCDates)}>
+                     <Icon type='date' size={13} classes="mr-1" /> {daysName(scFilter)}
+                  </span>
+                  {ShowSCDates && (
+                     <div className='absolute w-24 z-50 mt-0 right-0 bg-white border border-gray-200 rounded text-center'>
+                        {['threeDays', 'sevenDays', 'thirtyDays'].map((itemKey) => {
+                           return <button
+                                    key={itemKey}
+                                    className={`${scDataFilterStlye} ${scFilter === itemKey ? ' bg-indigo-100 text-indigo-600' : ''}`}
+                                    onClick={() => { setShowSCDates(false); if (setScFilter) setScFilter(itemKey); }}
+                                    >Last {daysName(itemKey)}
+                                 </button>;
+                        })}
+                     </div>
+                  )}
+               </div>
+            )}
+            {isIdeas && (
+               <button
+               data-testid="load_ideas"
+               className={'ml-2 text-blue-700 font-bold text-sm flex items-center lg:px-4 lg:py-2'}
+               onClick={() => showIdeaUpdateModal && showIdeaUpdateModal()}>
+                  <span
+                  className='text-center leading-4 mr-2 inline-block rounded-full w-7 h-7 pt-1 bg-blue-700 text-white font-bold text-lg'>
+                     <Icon type='reload' size={12} />
+                  </span>
+                  <i className=' not-italic hidden lg:inline-block'>Load Ideas</i>
+               </button>
+            )}
          </div>
       </div>
+      </div>
    );
 };
 

components/domains/DomainItem.tsx

@@ -0,0 +1,89 @@
+/* eslint-disable @next/next/no-img-element */
+// import { useRouter } from 'next/router';
+// import { useState } from 'react';
+import TimeAgo from 'react-timeago';
+import dayjs from 'dayjs';
+import Link from 'next/link';
+import Icon from '../common/Icon';
+
+type DomainItemProps = {
+   domain: DomainType,
+   selected: boolean,
+   isConsoleIntegrated: boolean,
+   thumb: string,
+   updateThumb: Function,
+}
+
+const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb, updateThumb }: DomainItemProps) => {
+   const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain;
+   // const router = useRouter();
+   return (
+      <div className={`domItem bg-white border rounded w-full text-sm mb-10 hover:border-indigo-200 ${selected ? '' : ''}`}>
+         <Link href={`/domain/${slug}`} passHref={true}>
+         <a className='flex flex-col lg:flex-row'>
+            <div className={`flex-1 p-6 flex ${!isConsoleIntegrated ? 'basis-1/3' : ''}`}>
+               <div className="group domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded
+                  border border-gray-200 overflow-hidden flex justify-center relative">
+                  <button
+                     className=' absolute right-1 top-0 text-gray-400 p-1 transition-all
+                     invisible opacity-0 group-hover:visible group-hover:opacity-100 hover:text-gray-600 z-10'
+                     title='Reload Website Screenshot'
+                     onClick={(e) => { e.preventDefault(); e.stopPropagation(); updateThumb(domain.domain); }}
+                  >
+                     <Icon type="reload" size={12} />
+                  </button>
+                  <img
+                  className={`self-center ${!thumb ? 'max-w-[50px]' : ''}`}
+                  src={thumb || `https://www.google.com/s2/favicons?domain=${domain.domain}&sz=128`} alt={domain.domain}
+                  />
+               </div>
+               <div className="domain_details flex-1">
+                  <h3 className='font-semibold text-base mb-2 max-w-[200px] text-ellipsis overflow-hidden' title={domain.domain}>{domain.domain}</h3>
+                 {keywordsUpdated && (
+                  <span className=' text-gray-600 text-xs'>
+                     Updated <TimeAgo title={dayjs(keywordsUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={keywordsUpdated} />
+                  </span>
+                 )}
+               </div>
+            </div>
+            <div className='flex-1 flex flex-col p-4'>
+               <div className=' bg-indigo-50 p-1 px-2 text-xs rounded-full absolute ml-3 mt-[-8px]'>
+                  <Icon type="tracking" size={13} color="#364aff" /> Tracker
+               </div>
+               <div className='dom_stats flex flex-1 font-semibold text-2xl p-4 pt-5 rounded border border-[#E9EBFF] text-center'>
+                  <div className="flex-1 relative">
+                     <span className='block text-xs lg:text-sm text-gray-500 mb-1'>Keywords</span>{keywordCount}
+                  </div>
+                  <div className="flex-1 relative">
+                     <span className='block text-xs lg:text-sm text-gray-500 mb-1'>Avg position</span>{avgPosition}
+                  </div>
+               </div>
+            </div>
+            {isConsoleIntegrated && (
+               <div className='flex-1 flex-col p-4 lg:basis-56'>
+                  <div className=' bg-indigo-50 p-1 px-2 text-xs rounded-full absolute ml-3 mt-[-8px]'>
+                     <Icon type="google" size={13} /> Search Console (7d)
+                  </div>
+                  <div className='dom_sc_stats flex flex-1 h-full font-semibold text-2xl p-4 pt-5 rounded border border-[#E9EBFF] text-center'>
+                     <div className="flex-1 relative">
+                        <span className='block text-xs lg:text-sm text-gray-500 mb-1'>Visits</span>
+                        {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(scVisits).replace('T', 'K')}
+                     </div>
+                     <div className="flex-1 relative">
+                        <span className='block text-xs lg:text-sm text-gray-500 mb-1'>Impressions</span>
+                        {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(scImpressions).replace('T', 'K')}
+                     </div>
+                     <div className="flex-1 relative">
+                        <span className='block text-xs lg:text-sm text-gray-500 mb-1'>Avg position</span>
+                        {scPosition}
+                     </div>
+                  </div>
+               </div>
+            )}
+         </a>
+         </Link>
+      </div>
+   );
+};
+
+export default DomainItem;

components/domains/DomainSettings.tsx

@@ -1,12 +1,13 @@
 import { useRouter } from 'next/router';
-import { useEffect, useState } from 'react';
+import { useState } from 'react';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
-import { useDeleteDomain, useUpdateDomain } from '../../services/domains';
+import { useDeleteDomain, useFetchDomain, useUpdateDomain } from '../../services/domains';
+import InputField from '../common/InputField';
+import SelectField from '../common/SelectField';
 
 type DomainSettingsProps = {
-   domain:Domain|false,
-   domains: Domain[],
+   domain:DomainType|false,
    closeModal: Function
 }
 
@@ -15,37 +16,33 @@ type DomainSettingsError = {
    msg: string,
 }
 
-const DomainSettings = ({ domain, domains, closeModal }: DomainSettingsProps) => {
+const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
    const router = useRouter();
+   const [currentTab, setCurrentTab] = useState<'notification'|'searchconsole'>('notification');
    const [showRemoveDomain, setShowRemoveDomain] = useState<boolean>(false);
    const [settingsError, setSettingsError] = useState<DomainSettingsError>({ type: '', msg: '' });
-   const [domainSettings, setDomainSettings] = useState<DomainSettings>({ notification_interval: 'never', notification_emails: '' });
+   const [domainSettings, setDomainSettings] = useState<DomainSettings>(() => ({
+      notification_interval: domain && domain.notification_interval ? domain.notification_interval : 'never',
+      notification_emails: domain && domain.notification_emails ? domain.notification_emails : '',
+      search_console: domain && domain.search_console ? JSON.parse(domain.search_console) : {
+         property_type: 'domain', url: '', client_email: '', private_key: '',
+      },
+   }));
 
-   const { mutate: updateMutate } = useUpdateDomain(() => closeModal(false));
-   const { mutate: deleteMutate } = useDeleteDomain(() => {
-      closeModal(false);
-      const fitleredDomains = domain && domains.filter((d:Domain) => d.domain !== domain.domain);
-      if (fitleredDomains && fitleredDomains[0] && fitleredDomains[0].slug) {
-         router.push(`/domain/${fitleredDomains[0].slug}`);
-      }
+   const { mutate: updateMutate, error: domainUpdateError, isLoading: isUpdating } = useUpdateDomain(() => closeModal(false));
+   const { mutate: deleteMutate } = useDeleteDomain(() => { closeModal(false); router.push('/domains'); });
+
+   // Get the Full Domain Data along with the Search Console API Data.
+   useFetchDomain(router, domain && domain.domain ? domain.domain : '', (domainObj:DomainType) => {
+      const currentSearchConsoleSettings = domainObj.search_console && JSON.parse(domainObj.search_console);
+      setDomainSettings({ ...domainSettings, search_console: currentSearchConsoleSettings || domainSettings.search_console });
    });
 
-   useEffect(() => {
-      if (domain) {
-         setDomainSettings({ notification_interval: domain.notification_interval, notification_emails: domain.notification_emails });
-      }
-   }, [domain]);
-
-   const updateNotiEmails = (event:React.FormEvent<HTMLInputElement>) => {
-      setDomainSettings({ ...domainSettings, notification_emails: event.currentTarget.value });
-   };
-
    const updateDomain = () => {
-      console.log('Domain: ');
       let error: DomainSettingsError | null = null;
       if (domainSettings.notification_emails) {
          const notification_emails = domainSettings.notification_emails.split(',');
-         const invalidEmails = notification_emails.find((x) => /^\w+([.-]?\w+)*@\w+([.-]?\w+)*(\.\w{2,3})+$/.test(x) === false);
+         const invalidEmails = notification_emails.find((x) => /^\w+([.-]?\w+)*@\w+([.-]?\w+)*(\.\w{2,15})+$/.test(x) === false);
          console.log('invalidEmails: ', invalidEmails);
          if (invalidEmails) {
             error = { type: 'email', msg: 'Invalid Email' };
@@ -62,24 +59,103 @@ const DomainSettings = ({ domain, domains, closeModal }: DomainSettingsProps) =>
          }
    };
 
+   const tabStyle = `inline-block px-4 py-2 rounded-md mr-3 cursor-pointer text-sm select-none z-10
+                     text-gray-600 border border-b-0 relative top-[1px] rounded-b-none`;
    return (
       <div>
-         <Modal closeModal={() => closeModal(false)} title={'Domain Settings'} width="[500px]">
+         <Modal closeModal={() => closeModal(false)} title={'Domain Settings'} width="[500px]" verticalCenter={currentTab === 'searchconsole'} >
             <div data-testid="domain_settings" className=" text-sm">
-               <div className="mb-6 flex justify-between items-center">
-                  <h4>Notification Emails
-                     {settingsError.type === 'email' && <span className="text-red-500 font-semibold ml-2">{settingsError.msg}</span>}
-                  </h4>
-                  <input
-                     className={`border w-46 text-sm transition-all rounded p-1.5 px-4 outline-none ring-0 
-                     ${settingsError.type === 'email' ? ' border-red-300' : ''}`}
-                     type="text"
-                     placeholder='Your Emails'
-                     onChange={updateNotiEmails}
-                     value={domainSettings.notification_emails || ''}
-                  />
+               <div className=' mt-3 mb-5 border  border-slate-200 px-2 py-4 pb-0
+               relative left-[-20px] w-[calc(100%+40px)] border-l-0 border-r-0 bg-[#f8f9ff]'>
+                  <ul>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'notification' ? ' bg-white text-blue-600 border-slate-200' : 'border-transparent'} `}
+                     onClick={() => setCurrentTab('notification')}>
+                       <Icon type='email' /> Notification
+                     </li>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'searchconsole' ? ' bg-white text-blue-600 border-slate-200' : 'border-transparent'}`}
+                     onClick={() => setCurrentTab('searchconsole')}>
+                        <Icon type='google' /> Search Console
+                     </li>
+                  </ul>
                </div>
+
+               <div>
+                  {currentTab === 'notification' && (
+                     <div className="mb-4 flex justify-between items-center w-full">
+                        <InputField
+                        label='Notification Emails'
+                        onChange={(emails:string) => setDomainSettings({ ...domainSettings, notification_emails: emails })}
+                        value={domainSettings.notification_emails || ''}
+                        placeholder='Your Emails'
+                        />
+                     </div>
+                  )}
+                  {currentTab === 'searchconsole' && (
+                     <>
+                        <div className="mb-4 flex justify-between items-center w-full">
+                           <label className='mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'>Property Type</label>
+                           <SelectField
+                           options={[{ label: 'Domain', value: 'domain' }, { label: 'URL', value: 'url' }]}
+                           selected={[domainSettings.search_console?.property_type || 'domain']}
+                           defaultLabel="Select Search Console Property Type"
+                           updateField={(updated:['domain'|'url']) => setDomainSettings({
+                              ...domainSettings,
+                              search_console: { ...(domainSettings.search_console as DomainSearchConsole), property_type: updated[0] || 'domain' },
+                           })}
+                           multiple={false}
+                           rounded={'rounded'}
+                           />
+                        </div>
+                        {domainSettings?.search_console?.property_type === 'url' && (
+                           <div className="mb-4 flex justify-between items-center w-full">
+                              <InputField
+                              label='Property URL (Required)'
+                              onChange={(url:string) => setDomainSettings({
+                                 ...domainSettings,
+                                 search_console: { ...(domainSettings.search_console as DomainSearchConsole), url },
+                              })}
+                              value={domainSettings?.search_console?.url || ''}
+                              placeholder='Search Console Property URL. eg: https://mywebsite.com/'
+                              />
+                           </div>
+                        )}
+                        <div className="mb-4 flex justify-between items-center w-full">
+                           <InputField
+                           label='Search Console Client Email'
+                           onChange={(client_email:string) => setDomainSettings({
+                              ...domainSettings,
+                              search_console: { ...(domainSettings.search_console as DomainSearchConsole), client_email },
+                           })}
+                           value={domainSettings?.search_console?.client_email || ''}
+                           placeholder='myapp@appspot.gserviceaccount.com'
+                           />
+                        </div>
+                        <div className="mb-4 flex flex-col justify-between items-center w-full">
+                           <label className='mb-2 font-semibold block text-sm text-gray-700 capitalize w-full'>Search Console Private Key</label>
+                           <textarea
+                              className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
+                              focus:outline-none h-[100px] focus:border-blue-200`}
+                              value={domainSettings?.search_console?.private_key || ''}
+                              placeholder={'-----BEGIN PRIVATE KEY-----/ssssaswdkihad....'}
+                              onChange={(event) => setDomainSettings({
+                                 ...domainSettings,
+                                 search_console: { ...(domainSettings.search_console as DomainSearchConsole), private_key: event.target.value },
+                              })}
+                           />
+                        </div>
+                     </>
+                  )}
+               </div>
+               {!isUpdating && (domainUpdateError as Error)?.message && (
+                  <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{(domainUpdateError as Error).message}</div>
+               )}
+               {!isUpdating && settingsError?.msg && (
+                  <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{settingsError.msg}</div>
+               )}
             </div>
+
             <div className="flex justify-between border-t-[1px] border-gray-100 mt-8 pt-4 pb-0">
                <button
                className="text-sm font-semibold text-red-500"
@@ -87,9 +163,9 @@ const DomainSettings = ({ domain, domains, closeModal }: DomainSettingsProps) =>
                   <Icon type="trash" /> Remove Domain
                </button>
                <button
-               className='text-sm font-semibold py-2 px-5 rounded cursor-pointer bg-blue-700 text-white'
-               onClick={() => updateDomain()}>
-                  Update Settings
+               className={`text-sm font-semibold py-2 px-5 rounded cursor-pointer bg-blue-700 text-white ${isUpdating ? 'cursor-not-allowed' : ''}`}
+               onClick={() => !isUpdating && updateDomain()}>
+                  {isUpdating && <Icon type='loading' />} Update Settings
                </button>
             </div>
          </Modal>

components/ideas/IdeaDetails.tsx

@@ -0,0 +1,139 @@
+import React, { useMemo, useRef } from 'react';
+import dayjs from 'dayjs';
+import { useQuery } from 'react-query';
+import { useRouter } from 'next/router';
+import Icon from '../common/Icon';
+import countries from '../../utils/countries';
+import Chart from '../common/Chart';
+import useOnKey from '../../hooks/useOnKey';
+import { formattedNum } from '../../utils/client/helpers';
+import { fetchSearchResults } from '../../services/keywords';
+
+type IdeaDetailsProps = {
+   keyword: IdeaKeyword,
+   closeDetails: Function
+}
+
+const dummySearchResults = [
+   { position: 1, url: 'https://google.com/?search=dummy+text', title: 'Google Search Result One' },
+   { position: 1, url: 'https://yahoo.com/?search=dummy+text', title: 'Yahoo Results | Sample Dummy' },
+   { position: 1, url: 'https://gamespot.com/?search=dummy+text', title: 'GameSpot | Dummy Search Results' },
+   { position: 1, url: 'https://compressimage.com/?search=dummy+text', title: 'Compress Images Online' },
+];
+
+const IdeaDetails = ({ keyword, closeDetails }:IdeaDetailsProps) => {
+   const router = useRouter();
+   const updatedDate = new Date(keyword.updated);
+   const searchResultContainer = useRef<HTMLDivElement>(null);
+   const searchResultFound = useRef<HTMLDivElement>(null);
+   const searchResultReqPayload = { keyword: keyword.keyword, country: keyword.country, device: 'desktop' };
+   const { data: keywordSearchResultData, refetch: fetchKeywordSearchResults, isLoading: fetchingResult } = useQuery(
+   `ideas:${keyword.uid}`,
+      () => fetchSearchResults(router, searchResultReqPayload),
+      { refetchOnWindowFocus: false, enabled: false },
+   );
+   const { monthlySearchVolumes } = keyword;
+
+   useOnKey('Escape', closeDetails);
+
+   const chartData = useMemo(() => {
+      const chartDataObj: { labels: string[], sreies: number[] } = { labels: [], sreies: [] };
+      Object.keys(monthlySearchVolumes).forEach((dateKey:string) => {
+         const dateKeyArr = dateKey.split('-');
+         const labelDate = `${dateKeyArr[0].slice(0, 1).toUpperCase()}${dateKeyArr[0].slice(1, 3).toLowerCase()}, ${dateKeyArr[1].slice(2)}`;
+         chartDataObj.labels.push(labelDate);
+         chartDataObj.sreies.push(parseInt(monthlySearchVolumes[dateKey], 10));
+      });
+      return chartDataObj;
+   }, [monthlySearchVolumes]);
+
+   const closeOnBGClick = (e:React.SyntheticEvent) => {
+      e.stopPropagation();
+      e.nativeEvent.stopImmediatePropagation();
+      if (e.target === e.currentTarget) { closeDetails(); }
+   };
+
+   const searchResultsFetched = !!keywordSearchResultData?.searchResult?.results;
+   const keywordSearchResult = searchResultsFetched ? keywordSearchResultData?.searchResult?.results : dummySearchResults;
+
+   return (
+       <div className="IdeaDetails fixed w-full h-screen top-0 left-0 z-[99999]" onClick={closeOnBGClick} data-testid="IdeaDetails">
+            <div className="IdeaDetails absolute w-full lg:w-5/12 bg-white customShadow top-0 right-0 h-screen" >
+               <div className='IdeaDetails__header p-6 border-b border-b-slate-200 text-slate-500'>
+                  <h3 className=' text-lg font-bold'>
+                     <span title={countries[keyword.country][0]}
+                     className={`fflag fflag-${keyword.country} w-[18px] h-[12px] mr-2`} /> {keyword.keyword}
+                     <span className='py-1 px-2 ml-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>
+                        {formattedNum(keyword.avgMonthlySearches)}/month
+                     </span>
+                  </h3>
+                  <button
+                  className='absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
+                  onClick={() => closeDetails()}>
+                     <Icon type='close' size={24} />
+                  </button>
+               </div>
+               <div className='IdeaDetails__content p-6'>
+
+                  <div className='IdeaDetails__section'>
+                     <div className="IdeaDetails__section__head flex justify-between mb-5">
+                        <h3 className=' font-bold text-gray-700 text-lg'>Search Volume Trend</h3>
+                     </div>
+                     <div className='IdeaDetails__section__chart h-64'>
+                        <Chart labels={chartData.labels} sreies={chartData.sreies} noMaxLimit={true} reverse={false} />
+                     </div>
+                  </div>
+                  <div className='IdeaDetails__section mt-10'>
+                     <div className="IdeaDetails__section__head flex justify-between items-center pb-4 mb-4 border-b border-b-slate-200">
+                        <h3 className=' font-bold text-gray-700 lg:text-lg'>Google Search Result
+                           <a className='text-gray-400 hover:text-indigo-600 inline-block ml-1 px-2 py-1'
+                              href={`https://www.google.com/search?q=${encodeURI(keyword.keyword)}`}
+                              target="_blank"
+                              rel='noreferrer'>
+                              <Icon type='link' size={14} />
+                           </a>
+                        </h3>
+                        <span className=' text-xs text-gray-500'>{dayjs(updatedDate).format('MMMM D, YYYY')}</span>
+                     </div>
+                     <div className={'keywordDetails__section__results styled-scrollbar overflow-y-auto relative'} ref={searchResultContainer}>
+                        {!searchResultsFetched && (
+                           <div className=' absolute flex w-full h-full justify-center items-center flex-col z-50 font-semibold'>
+                              <p>View Google Search Results for &quot;{keyword.keyword}&quot;</p>
+                              <button
+                              onClick={() => fetchKeywordSearchResults()}
+                              className='mt-4 text-sm font-semibold py-2 px-5 rounded cursor-pointer bg-blue-700 text-white '>
+                                 <Icon type={fetchingResult ? 'loading' : 'google'} /> {fetchingResult ? 'Performing' : 'Perform'}  Google Search
+                              </button>
+                           </div>
+                        )}
+                        <div className={`${!searchResultsFetched ? ' blur-sm ' : ''}`}>
+                           {keywordSearchResult && Array.isArray(keywordSearchResult) && keywordSearchResult.length > 0 && (
+                              keywordSearchResult.map((item, index) => {
+                                 const { position } = keyword;
+                                 const domainExist = position < 100 && index === (position - 1);
+                                 return (
+                                    <div
+                                    ref={domainExist ? searchResultFound : null}
+                                    className={`leading-6 mb-4 mr-3 p-3 text-sm break-all pr-3 rounded 
+                                    ${domainExist ? ' bg-amber-50 border border-amber-200' : ''}`}
+                                    key={item.url + item.position}>
+                                       <h4 className='font-semibold text-blue-700'>
+                                          <a href={item.url} target="_blank" rel='noreferrer'>{`${index + 1}. ${item.title}`}</a>
+                                       </h4>
+                                       {/* <p>{item.description}</p> */}
+                                       <a className=' text-green-900' href={item.url} target="_blank" rel='noreferrer'>{item.url}</a>
+                                    </div>
+                                 );
+                              })
+                           )}
+                        </div>
+                     </div>
+
+                  </div>
+               </div>
+            </div>
+       </div>
+   );
+};
+
+export default IdeaDetails;

components/ideas/IdeasFilter.tsx

@@ -0,0 +1,135 @@
+import React, { useState } from 'react';
+import Icon from '../common/Icon';
+import SelectField, { SelectionOption } from '../common/SelectField';
+
+type IdeasFilterProps = {
+   allTags: string[],
+   filterParams: KeywordFilters,
+   filterKeywords: Function,
+   keywords: IdeaKeyword[],
+   favorites: IdeaKeyword[],
+   updateSort: Function,
+   showFavorites: Function,
+   sortBy: string,
+}
+
+const IdeasFilters = (props: IdeasFilterProps) => {
+   const { filterKeywords, allTags = [], updateSort, showFavorites, sortBy, filterParams, keywords = [], favorites = [] } = props;
+   const [keywordType, setKeywordType] = useState<'all'|'favorites'>('all');
+   const [sortOptions, showSortOptions] = useState(false);
+   const [filterOptions, showFilterOptions] = useState(false);
+
+   const filterTags = (tags:string[]) => filterKeywords({ ...filterParams, tags });
+
+   const searchKeywords = (event:React.FormEvent<HTMLInputElement>) => {
+      const filtered = filterKeywords({ ...filterParams, search: event.currentTarget.value });
+      return filtered;
+   };
+
+   const sortOptionChoices: SelectionOption[] = [
+      { value: 'alpha_asc', label: 'Alphabetically(A-Z)' },
+      { value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
+      { value: 'vol_asc', label: 'Lowest Search Volume' },
+      { value: 'vol_desc', label: 'Highest Search Volume' },
+      { value: 'competition_asc', label: 'High Competition' },
+      { value: 'competition_desc', label: 'Low Competition' },
+   ];
+
+   const sortItemStyle = (sortType:string) => {
+      return `cursor-pointer py-2 px-3 hover:bg-[#FCFCFF] ${sortBy === sortType ? 'bg-indigo-50 text-indigo-600 hover:bg-indigo-50' : ''}`;
+   };
+
+   const deviceTabStyle = 'select-none cursor-pointer px-3 py-2 rounded-3xl mr-2';
+   const deviceTabCountStyle = 'px-2 py-0 rounded-3xl bg-[#DEE1FC] text-[0.7rem] font-bold ml-1';
+   const mobileFilterOptionsStyle = 'visible mt-8 border absolute min-w-[0] rounded-lg max-h-96 bg-white z-50 w-52 right-2 p-4';
+
+   return (
+      <div className='domKeywords_filters py-4 px-6 flex justify-between text-sm text-gray-500 font-semibold border-b-[1px] lg:border-0 items-center'>
+         <div>
+            <ul className='flex text-xs'>
+               <li
+                data-testid="desktop_tab"
+               className={`${deviceTabStyle} ${keywordType === 'all' ? ' bg-[#F8F9FF] text-gray-700' : ''}`}
+               onClick={() => { setKeywordType('all'); showFavorites(false); }}>
+                     <Icon type='keywords' classes='top-[3px]' size={15} />
+                     <i className='hidden not-italic lg:inline-block ml-1'>All Keywords</i>
+                     <span className={`${deviceTabCountStyle}`}>{keywords.length}</span>
+               </li>
+               <li
+               data-testid="mobile_tab"
+               className={`${deviceTabStyle} ${keywordType === 'favorites' ? ' bg-[#F8F9FF] text-gray-700' : ''}`}
+               onClick={() => { setKeywordType('favorites'); showFavorites(true); }}>
+                     <Icon type='star' classes='top-[4px]' />
+                     <i className='hidden not-italic lg:inline-block ml-1'>Favorites</i>
+                     <span className={`${deviceTabCountStyle}`}>{favorites.length}</span>
+               </li>
+            </ul>
+         </div>
+
+            <div className='flex gap-5'>
+
+               <div className=' lg:hidden'>
+                  <button
+                  data-testid="filter_button"
+                  className={`px-2 py-1 rounded ${filterOptions ? ' bg-indigo-100 text-blue-700' : ''}`}
+                  title='Filter'
+                  onClick={() => showFilterOptions(!filterOptions)}>
+                     <Icon type="filter" size={18} />
+                  </button>
+               </div>
+               <div className={`lg:flex gap-5 lg:visible ${filterOptions ? mobileFilterOptionsStyle : 'hidden'}`}>
+                  {keywordType === 'all' && (
+                     <div className={'tags_filter mb-2 lg:mb-0'}>
+                        <SelectField
+                           selected={filterParams.tags}
+                           options={allTags.map((tag:string) => ({ label: tag, value: tag }))}
+                           defaultLabel={`All Groups (${allTags.length})`}
+                           updateField={(updated:string[]) => filterTags(updated)}
+                           emptyMsg="No Groups Found for this Domain"
+                           minWidth={270}
+                        />
+                     </div>
+                  )}
+                  <div className={'mb-2 lg:mb-0'}>
+                     <input
+                        data-testid="filter_input"
+                        className={`border w-44 lg:w-36 focus:w-44 transition-all rounded-3xl 
+                        p-1.5 px-4 outline-none ring-0 focus:border-indigo-200`}
+                        type="text"
+                        placeholder='Filter Keywords...'
+                        onChange={searchKeywords}
+                        value={filterParams.search}
+                     />
+                  </div>
+               </div>
+               <div className='relative'>
+                  <button
+                  data-testid="sort_button"
+                  className={`px-2 py-1 rounded ${sortOptions ? ' bg-indigo-100 text-blue-700' : ''}`}
+                  title='Sort'
+                  onClick={() => showSortOptions(!sortOptions)}>
+                     <Icon type="sort" size={18} />
+                  </button>
+                  {sortOptions && (
+                     <ul
+                     data-testid="sort_options"
+                     className='sort_options mt-2 border absolute min-w-[0] right-0 rounded-lg
+                     max-h-96 bg-white z-[9999] w-44 overflow-y-auto styled-scrollbar'>
+                        {sortOptionChoices.map((sortOption) => {
+                           return <li
+                                    key={sortOption.value}
+                                    className={sortItemStyle(sortOption.value)}
+                                    onClick={() => { updateSort(sortOption.value); showSortOptions(false); }}>
+                                       {sortOption.label}
+                                    </li>;
+                        })}
+                     </ul>
+                  )}
+               </div>
+            </div>
+
+      </div>
+   );
+};
+
+export default IdeasFilters;

components/ideas/KeywordIdea.tsx

@@ -0,0 +1,77 @@
+import React, { useMemo } from 'react';
+import Icon from '../common/Icon';
+import countries from '../../utils/countries';
+import { formattedNum } from '../../utils/client/helpers';
+import ChartSlim from '../common/ChartSlim';
+
+type KeywordIdeaProps = {
+   keywordData: IdeaKeyword,
+   selected: boolean,
+   lastItem?:boolean,
+   isFavorite: boolean,
+   style: Object,
+   selectKeyword: Function,
+   favoriteKeyword:Function,
+   showKeywordDetails: Function
+}
+
+const KeywordIdea = (props: KeywordIdeaProps) => {
+   const { keywordData, selected, lastItem, selectKeyword, style, isFavorite = false, favoriteKeyword, showKeywordDetails } = props;
+   const { keyword, uid, position, country, monthlySearchVolumes, avgMonthlySearches, competition, competitionIndex } = keywordData;
+
+   const chartData = useMemo(() => {
+      const chartDataObj: { labels: string[], sreies: number[] } = { labels: [], sreies: [] };
+      Object.keys(monthlySearchVolumes).forEach((dateKey:string) => {
+         chartDataObj.labels.push(dateKey);
+         chartDataObj.sreies.push(parseInt(monthlySearchVolumes[dateKey], 10));
+      });
+      return chartDataObj;
+   }, [monthlySearchVolumes]);
+
+   return (
+      <div
+      key={keyword}
+      style={style}
+      className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
+      lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
+
+         <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold cursor-pointer'>
+            <button
+               className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
+               ${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
+               onClick={() => selectKeyword(uid)}
+               >
+                  <Icon type="check" size={10} />
+            </button>
+            <a className='py-2 hover:text-blue-600' onClick={() => showKeywordDetails()}>
+               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country] && countries[country][0]} />{keyword}
+            </a>
+            <button
+            className={`ml-2 hover:text-yellow-600 hover:opacity-100 ${isFavorite ? 'text-yellow-600' : ' opacity-50'}`}
+            onClick={() => favoriteKeyword()}>
+               <Icon type={isFavorite ? 'star-filled' : 'star'} classes='top-[4px]' size={18} />
+            </button>
+         </div>
+
+         <div className='keyword_imp text-center inline-block ml-6 lg:ml-0 lg:flex-1 '>
+            {formattedNum(avgMonthlySearches)}<span className='lg:hidden'>/month</span>
+         </div>
+
+         <div
+          onClick={() => showKeywordDetails()}
+         className={`keyword_visits text-center hidden mt-4 mr-5 ml-5 cursor-pointer
+         lg:flex-1 lg:m-0 lg:ml-10 max-w-[70px] lg:max-w-none lg:pr-5 lg:flex justify-center`}>
+            {chartData.labels.length > 0 && <ChartSlim labels={chartData.labels} sreies={chartData.sreies} noMaxLimit={true} reverse={false} />}
+         </div>
+
+         <div className='keyword_ctr text-center inline-block ml-4 lg:flex mt-4 relative lg:flex-1 lg:m-0 justify-center'>
+            <div className={`idea_competiton idea_competiton--${competition} flex bg-slate-100 rounded w-28 text-xs font-semibold`}>
+               <span className=' inline-block p-1 flex-1'>{competitionIndex}/100</span>
+               <span className=' inline-block p-1 flex-1 rounded-e text-white'>{competition}</span>
+            </div>
+         </div>
+      </div>
+   );
+ };
+
+ export default KeywordIdea;

components/ideas/KeywordIdeasTable.tsx

@@ -0,0 +1,247 @@
+import { useRouter } from 'next/router';
+import React, { useState, useMemo } from 'react';
+import { Toaster } from 'react-hot-toast';
+import { useQuery } from 'react-query';
+import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
+import { useAddKeywords } from '../../services/keywords';
+import Icon from '../common/Icon';
+import KeywordIdea from './KeywordIdea';
+import useWindowResize from '../../hooks/useWindowResize';
+import useIsMobile from '../../hooks/useIsMobile';
+import { IdeasSortKeywords, IdeasfilterKeywords } from '../../utils/client/IdeasSortFilter';
+import IdeasFilters from './IdeasFilter';
+import { useMutateFavKeywordIdeas } from '../../services/adwords';
+import IdeaDetails from './IdeaDetails';
+import { fetchDomains } from '../../services/domains';
+import SelectField from '../common/SelectField';
+
+type IdeasKeywordsTableProps = {
+   domain: DomainType | null,
+   keywords: IdeaKeyword[],
+   favorites: IdeaKeyword[],
+   noIdeasDatabase: boolean,
+   isLoading: boolean,
+   showFavorites: boolean,
+   setShowFavorites: Function,
+   isAdwordsIntegrated: boolean,
+}
+
+const IdeasKeywordsTable = ({
+   domain, keywords = [], favorites = [], isLoading = true, isAdwordsIntegrated = true, setShowFavorites,
+   showFavorites = false, noIdeasDatabase = false }: IdeasKeywordsTableProps) => {
+   const router = useRouter();
+   const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
+   const [showKeyDetails, setShowKeyDetails] = useState<IdeaKeyword|null>(null);
+   const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
+   const [sortBy, setSortBy] = useState<string>('imp_desc');
+   const [listHeight, setListHeight] = useState(500);
+   const [addKeywordDevice, setAddKeywordDevice] = useState<'desktop'|'mobile'>('desktop');
+   const [addKeywordDomain, setAddKeywordDomain] = useState('');
+   const { mutate: addKeywords } = useAddKeywords(() => { if (domain && domain.slug) router.push(`/domain/${domain.slug}`); });
+   const { mutate: faveKeyword, isLoading: isFaving } = useMutateFavKeywordIdeas(router);
+   const [isMobile] = useIsMobile();
+   const isResearchPage = router.pathname === '/research';
+
+   const { data: domainsData } = useQuery('domains', () => fetchDomains(router, false), { enabled: selectedKeywords.length > 0, retry: false });
+   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
+
+   useWindowResize(() => setListHeight(window.innerHeight - (isMobile ? 200 : 400)));
+
+   const finalKeywords: IdeaKeyword[] = useMemo(() => {
+      const filteredKeywords = IdeasfilterKeywords(showFavorites ? favorites : keywords, filterParams);
+      const sortedKeywords = IdeasSortKeywords(filteredKeywords, sortBy);
+      return sortedKeywords;
+   }, [keywords, showFavorites, favorites, filterParams, sortBy]);
+
+   const favoriteIDs: string[] = useMemo(() => favorites.map((fav) => fav.uid), [favorites]);
+
+   const allTags:string[] = useMemo(() => {
+      const wordTags: Map<string, number> = new Map();
+      keywords.forEach((k) => {
+         const keywordsArray = k.keyword.split(' ');
+         const keywordFirstTwoWords = keywordsArray.slice(0, 2).join(' ');
+         const keywordFirstTwoWordsReversed = keywordFirstTwoWords.split(' ').reverse().join(' ');
+         if (!wordTags.has(keywordFirstTwoWordsReversed)) {
+            wordTags.set(keywordFirstTwoWords, 0);
+         }
+      });
+      [...wordTags].forEach((tag) => {
+         const foundTags = keywords.filter((kw) => kw.keyword.includes(tag[0]) || kw.keyword.includes(tag[0].split(' ').reverse().join(' ')));
+         if (foundTags.length < 3) {
+            wordTags.delete(tag[0]);
+         } else {
+            wordTags.set(tag[0], foundTags.length);
+         }
+      });
+      const finalWordTags = [...wordTags].sort((a, b) => (a[1] > b[1] ? -1 : 1)).map((t) => `${t[0]} (${t[1]})`);
+      return finalWordTags;
+   }, [keywords]);
+
+   const selectKeyword = (keywordID: string) => {
+      let updatedSelectd = [...selectedKeywords, keywordID];
+      if (selectedKeywords.includes(keywordID)) {
+         updatedSelectd = selectedKeywords.filter((keyID) => keyID !== keywordID);
+      }
+      setSelectedKeywords(updatedSelectd);
+   };
+
+   const favoriteKeyword = (keywordID: string) => {
+      if (!isFaving) {
+         faveKeyword({ keywordID, domain: isResearchPage ? 'research' : domain?.slug });
+      }
+   };
+
+   const addKeywordIdeasToTracker = () => {
+      const selectedkeywords:KeywordAddPayload[] = [];
+      keywords.forEach((kitem:IdeaKeyword) => {
+         if (selectedKeywords.includes(kitem.uid)) {
+            const { keyword, country } = kitem;
+            selectedkeywords.push({
+               keyword,
+               device: addKeywordDevice,
+               country,
+               domain: isResearchPage ? addKeywordDomain : (domain?.domain || ''),
+               tags: '',
+            });
+         }
+      });
+      addKeywords(selectedkeywords);
+      setSelectedKeywords([]);
+   };
+
+   const selectedAllItems = selectedKeywords.length === finalKeywords.length;
+
+   const Row = ({ data, index, style }:ListChildComponentProps) => {
+      const keyword: IdeaKeyword = data[index];
+      return (
+         <KeywordIdea
+         key={keyword.uid}
+         style={style}
+         selected={selectedKeywords.includes(keyword.uid)}
+         selectKeyword={selectKeyword}
+         favoriteKeyword={() => favoriteKeyword(keyword.uid)}
+         showKeywordDetails={() => setShowKeyDetails(keyword)}
+         isFavorite={favoriteIDs.includes(keyword.uid)}
+         keywordData={keyword}
+         lastItem={index === (finalKeywords.length - 1)}
+         />
+      );
+   };
+
+   return (
+      <div>
+         <div className='domKeywords flex flex-col bg-[white] rounded-md text-sm border mb-5'>
+            {selectedKeywords.length > 0 && (
+               <div className='font-semibold text-sm py-4 px-8 text-gray-500 '>
+                  <div className={`inline-block ${isResearchPage ? ' mr-2' : ''}`}>Add Keywords to Tracker</div>
+                  {isResearchPage && (
+                     <SelectField
+                     selected={[]}
+                     options={theDomains.map((d) => ({ label: d.domain, value: d.domain }))}
+                     defaultLabel={'Select a Domain'}
+                     updateField={(updated:string[]) => updated[0] && setAddKeywordDomain(updated[0])}
+                     emptyMsg="No Domains Found"
+                     multiple={false}
+                     inline={true}
+                     rounded='rounded'
+                     />
+                  )}
+                  <div className='inline-block ml-2'>
+                     <button
+                     className={`inline-block px-2 py-1 rounded-s 
+                     ${addKeywordDevice === 'desktop' ? 'bg-indigo-100 text-blue-700' : 'bg-indigo-50 '}`}
+                     onClick={() => setAddKeywordDevice('desktop')}>
+                        {addKeywordDevice === 'desktop' ? '◉' : '○'} Desktop
+                     </button>
+                     <button
+                     className={`inline-block px-2 py-1 rounded-e ${addKeywordDevice === 'mobile' ? 'bg-indigo-100 text-blue-700' : 'bg-indigo-50 '}`}
+                     onClick={() => setAddKeywordDevice('mobile')}>
+                        {addKeywordDevice === 'mobile' ? '◉' : '○'} Mobile
+                     </button>
+                  </div>
+                  <a
+                  className='inline-block px-2 py-2 cursor-pointer hover:text-indigo-600'
+                  onClick={() => addKeywordIdeasToTracker()}
+                  >
+                     <span className=' text-white bg-blue-700 px-2 py-1 rounded font-semibold'>+ Add Keywords</span>
+                  </a>
+               </div>
+            )}
+            {selectedKeywords.length === 0 && (
+               <IdeasFilters
+                  allTags={allTags}
+                  filterParams={filterParams}
+                  filterKeywords={(params:KeywordFilters) => setFilterParams(params)}
+                  updateSort={(sorted:string) => setSortBy(sorted)}
+                  sortBy={sortBy}
+                  keywords={keywords}
+                  favorites={favorites}
+                  showFavorites={(show:boolean) => { setShowFavorites(show); }}
+               />
+            )}
+            <div className='domkeywordsTable domkeywordsTable--sckeywords styled-scrollbar w-full overflow-auto min-h-[60vh]'>
+               <div className=' lg:min-w-[800px]'>
+                  <div className={`domKeywords_head domKeywords_head--${sortBy} hidden lg:flex p-3 px-6 bg-[#FCFCFF]
+                   text-gray-600 justify-between items-center font-semibold border-y`}>
+                     <span className='domKeywords_head_keyword flex-1 basis-20 w-auto '>
+                     {finalKeywords.length > 0 && (
+                        <button
+                           className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px]  border border-slate-300 
+                           ${selectedAllItems ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
+                           onClick={() => setSelectedKeywords(selectedAllItems ? [] : finalKeywords.map((k: IdeaKeyword) => k.uid))}
+                           >
+                              <Icon type="check" size={10} />
+                        </button>
+                     )}
+                        Keyword
+                     </span>
+                     <span className='domKeywords_head_vol flex-1 text-center'>Monthly Search</span>
+                     <span className='domKeywords_head_trend flex-1 text-center'>Search Trend</span>
+                     <span className='domKeywords_head_competition flex-1 text-center'>Competition</span>
+                  </div>
+                  <div className='domKeywords_keywords border-gray-200 min-h-[55vh] relative' data-domain={domain?.domain}>
+                     {!isLoading && finalKeywords && finalKeywords.length > 0 && (
+                        <List
+                        innerElementType="div"
+                        itemData={finalKeywords}
+                        itemCount={finalKeywords.length}
+                        itemSize={isMobile ? 100 : 57}
+                        height={listHeight}
+                        width={'100%'}
+                        className={'styled-scrollbar'}
+                        >
+                           {Row}
+                        </List>
+                     )}
+
+                     {isAdwordsIntegrated && isLoading && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>Loading Keywords Ideas...</p>
+                     )}
+                     {isAdwordsIntegrated && noIdeasDatabase && !isLoading && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>
+                           {'No keyword Ideas has been generated for this domain yet. Click the "Load Ideas" button to generate keyword ideas.'}
+                        </p>
+                     )}
+                     {isAdwordsIntegrated && !isLoading && finalKeywords.length === 0 && !noIdeasDatabase && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>
+                           {'No Keyword Ideas found. Please try generating Keyword Ideas again by clicking the "Load Ideas" button.'}
+                        </p>
+                     )}
+                     {!isAdwordsIntegrated && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>
+                           Google Ads has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-ads' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Ads.
+                        </p>
+                     )}
+                  </div>
+               </div>
+            </div>
+         </div>
+         {showKeyDetails && showKeyDetails.uid && (
+            <IdeaDetails keyword={showKeyDetails} closeDetails={() => setShowKeyDetails(null)} />
+         )}
+         <Toaster position='bottom-center' containerClassName="react_toaster" />
+      </div>
+   );
+ };
+
+ export default IdeasKeywordsTable;

components/ideas/KeywordIdeasUpdater.tsx

@@ -0,0 +1,139 @@
+import { useMemo, useState } from 'react';
+import { useRouter } from 'next/router';
+import { useMutateKeywordIdeas } from '../../services/adwords';
+import allCountries, { adwordsLanguages } from '../../utils/countries';
+import SelectField from '../common/SelectField';
+import Icon from '../common/Icon';
+
+interface KeywordIdeasUpdaterProps {
+   onUpdate?: Function,
+   domain?: DomainType,
+   searchConsoleConnected: boolean,
+   adwordsConnected: boolean,
+   settings?: {
+      seedSCKeywords: boolean,
+      seedCurrentKeywords: boolean,
+      seedDomain: boolean,
+      language: string,
+      countries: string[],
+      keywords: string,
+      seedType: string
+   }
+}
+
+const KeywordIdeasUpdater = ({ onUpdate, settings, domain, searchConsoleConnected = false, adwordsConnected = false }: KeywordIdeasUpdaterProps) => {
+   const router = useRouter();
+   const [seedType, setSeedType] = useState(() => settings?.seedType || 'auto');
+   const [language, setLanguage] = useState(() => settings?.language.toString() || '1000');
+   const [countries, setCoutries] = useState<string[]>(() => settings?.countries || ['US']);
+   const [keywords, setKeywords] = useState(() => (settings?.keywords && Array.isArray(settings?.keywords) ? settings?.keywords.join(',') : ''));
+   const { mutate: updateKeywordIdeas, isLoading: isUpdatingIdeas } = useMutateKeywordIdeas(router, () => onUpdate && onUpdate());
+
+   const seedTypeOptions = useMemo(() => {
+      const options = [
+        { label: 'Automatically from Website Content', value: 'auto' },
+        { label: 'Based on currently tracked keywords', value: 'tracking' },
+        { label: 'From Custom Keywords', value: 'custom' },
+      ];
+
+      if (searchConsoleConnected) {
+        options.splice(-2, 0, { label: 'Based on already ranking keywords (GSC)', value: 'searchconsole' });
+      }
+
+      return options;
+    }, [searchConsoleConnected]);
+
+   const reloadKeywordIdeas = () => {
+      const keywordPaylod = seedType !== 'auto' && keywords ? keywords.split(',').map((key) => key.trim()) : undefined;
+      console.log('keywordPaylod :', keywords, keywordPaylod);
+      updateKeywordIdeas({
+         seedType,
+         language,
+         domain: domain?.domain,
+         domainSlug: domain?.slug,
+         keywords: keywordPaylod,
+         country: countries[0],
+      });
+   };
+
+   const countryOptions = useMemo(() => {
+      return Object.keys(allCountries)
+      .filter((countryISO) => allCountries[countryISO][3] !== 0)
+      .map((countryISO) => ({ label: allCountries[countryISO][0], value: countryISO }));
+   }, []);
+
+   const languageOPtions = useMemo(() => Object.entries(adwordsLanguages).map(([value, label]) => ({ label, value })), []);
+
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize w-full';
+   return (
+      <div>
+
+            <div>
+               <div className={'mb-3'}>
+                  <label className={labelStyle}>Get Keyword Ideas</label>
+                  <SelectField
+                     selected={[seedType]}
+                     options={seedTypeOptions}
+                     defaultLabel='Get Ideas Based On'
+                     updateField={(updated:string[]) => setSeedType(updated[0])}
+                     fullWidth={true}
+                     multiple={false}
+                     rounded='rounded'
+                  />
+               </div>
+               {seedType === 'custom' && (
+                  <>
+                     <div className={'mb-3'}>
+                        <label className={labelStyle}>Get Ideas from given Keywords (Max 20)</label>
+                        <textarea
+                        className='w-full border border-solid border-gray-300 focus:border-blue-100 p-3 rounded outline-none'
+                        value={keywords}
+                        onChange={(event) => setKeywords(event.target.value)}
+                        placeholder="keyword1, keyword2.."
+                        />
+                     </div>
+                     <hr className=' my-4' />
+                  </>
+               )}
+
+               <div className={'mb-3'}>
+                  <label className={labelStyle}>Country</label>
+                  <SelectField
+                     selected={countries}
+                     options={countryOptions}
+                     defaultLabel='All Countries'
+                     updateField={(updated:string[]) => setCoutries(updated)}
+                     flags={true}
+                     multiple={false}
+                     fullWidth={true}
+                     maxHeight={48}
+                     rounded='rounded'
+                  />
+               </div>
+               <div className={'mb-3'}>
+                  <label className={labelStyle}>Language</label>
+                  <SelectField
+                     selected={[language]}
+                     options={languageOPtions}
+                     defaultLabel='All Languages'
+                     updateField={(updated:string[]) => setLanguage(updated[0])}
+                     rounded='rounded'
+                     multiple={false}
+                     fullWidth={true}
+                     maxHeight={48}
+                  />
+               </div>
+
+               <button
+                  className={`w-full py-2 px-5 mt-2 rounded bg-blue-700 text-white 
+                  font-semibold ${!adwordsConnected ? ' cursor-not-allowed opacity-40' : 'cursor-pointer'}`}
+                  title={!adwordsConnected ? 'Please Connect Ads account to generate Keyword Ideas..' : ''}
+                  onClick={() => !isUpdatingIdeas && adwordsConnected && reloadKeywordIdeas()}>
+                     <Icon type={isUpdatingIdeas ? 'loading' : 'reload'} size={12} /> {isUpdatingIdeas ? 'Loading....' : 'Load Keyword Ideas'}
+               </button>
+            </div>
+      </div>
+   );
+};
+
+export default KeywordIdeasUpdater;

components/insight/Insight.tsx

@@ -0,0 +1,132 @@
+import React, { useState } from 'react';
+import { Toaster } from 'react-hot-toast';
+import { sortInsightItems } from '../../utils/insight';
+import SelectField from '../common/SelectField';
+import InsightItem from './InsightItem';
+import InsightStats from './InsightStats';
+
+type SCInsightProps = {
+   domain: DomainType | null,
+   insight: InsightDataType,
+   isLoading: boolean,
+   isConsoleIntegrated: boolean,
+}
+
+const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true, domain }: SCInsightProps) => {
+   const [activeTab, setActiveTab] = useState<string>('stats');
+
+   const insightItems = insight[activeTab as keyof InsightDataType];
+   const startDate = insight && insight.stats && insight.stats.length > 0 ? new Date(insight.stats[0].date) : null;
+   const endDate = insight && insight.stats && insight.stats.length > 0 ? new Date(insight.stats[insight.stats.length - 1].date) : null;
+
+   const switchTab = (tab: string) => {
+      // window.insightTab = tab;
+      setActiveTab(tab);
+   };
+
+   const renderTableHeader = () => {
+      const headerNames: {[key:string]: string[]} = {
+         stats: ['Date', 'Avg Position', 'Visits', 'Impressions', 'CTR'],
+         keywords: ['Keyword', 'Avg Position', 'Visits ↑', 'Impressions', 'CTR', 'Countries'],
+         countries: ['Country', 'Avg Position', 'Visits ↑', 'Impressions', 'CTR', 'Keywords'],
+         pages: ['Page', 'Avg Position', 'Visits ↑', 'Impressions', 'CTR', 'Countries', 'Keywords'],
+      };
+
+      return (
+         <div className={`domKeywords_head hidden lg:flex p-3 px-6 bg-[#FCFCFF]
+            text-gray-600 justify-between items-center font-semibold border-y`}>
+            <span className='domKeywords_head_keyword flex-1 basis-20 w-auto '>{headerNames[activeTab][0]}</span>
+            <span className='domKeywords_head_position flex-1 basis-40 grow-0 text-center'>{headerNames[activeTab][1]}</span>
+            <span className='domKeywords_head_imp flex-1 text-center'>{headerNames[activeTab][2]}</span>
+            <span className='domKeywords_head_visits flex-1 text-center'>{headerNames[activeTab][3]}</span>
+            <span className='domKeywords_head_ctr flex-1 text-center'>{headerNames[activeTab][4]}</span>
+            {headerNames[activeTab][5] && <span className='domKeywords_head_ctr flex-1 text-center'>{headerNames[activeTab][5]}</span>}
+            {headerNames[activeTab][6] && <span className='domKeywords_head_ctr flex-1 text-center'>{headerNames[activeTab][6]}</span>}
+         </div>
+      );
+   };
+
+   const deviceTabStyle = 'select-none cursor-pointer px-3 py-2 rounded-3xl mr-2';
+   const deviceTabCountStyle = 'px-2 py-0 rounded-3xl bg-[#DEE1FC] text-[0.7rem] font-bold ml-1';
+
+   return (
+      <div>
+         <div className='domKeywords flex flex-col bg-[white] rounded-md text-sm border mb-5'>
+            <div className='domKeywords_filters py-4 px-6 flex flex-col justify-between
+            text-sm text-gray-500 font-semibold border-b-[1px] lg:border-0 lg:flex-row'>
+               <div>
+                  <ul className='text-xs hidden lg:flex'>
+                     {['stats', 'keywords', 'countries', 'pages'].map((tabItem) => {
+                        const tabInsightItem = insight[tabItem as keyof InsightDataType];
+                        return <li
+                        key={`tab-${tabItem}`}
+                        className={`${deviceTabStyle} ${activeTab === tabItem ? ' bg-[#F8F9FF] text-gray-700' : ''}`}
+                        onClick={() => switchTab(tabItem)}>
+                              <i className='hidden not-italic lg:inline-block ml-1 capitalize'>{tabItem}</i>
+                              {tabItem !== 'stats' && (
+                                 <span className={`${deviceTabCountStyle}`}>
+                                    {tabInsightItem && tabInsightItem.length ? tabInsightItem.length : 0}
+                                 </span>
+                              )}
+                        </li>;
+                     })}
+                  </ul>
+                  <div className='insight_selector lg:hidden'>
+                     <SelectField
+                     options={['stats', 'keywords', 'countries', 'pages'].map((d) => { return { label: d, value: d }; })}
+                     selected={[activeTab]}
+                     defaultLabel="Select Tab"
+                     updateField={(updatedTab:[string]) => switchTab(updatedTab[0])}
+                     multiple={false}
+                     rounded={'rounded'}
+                     />
+                  </div>
+               </div>
+               {isConsoleIntegrated && (<div className='py-2 text-xs text-center mt-2 lg:text-sm lg:mt-0'>
+                  {startDate && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(startDate))}
+                  <span className='px-2 inline-block'>-</span>
+                  {endDate && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(endDate))}
+                  <span className='ml-2'>(Last 30 Days)</span>
+               </div>
+               )}
+            </div>
+            {isConsoleIntegrated && activeTab === 'stats' && (
+               <InsightStats
+               stats={insight?.stats ? insight.stats : []}
+               totalKeywords={insight?.keywords?.length || 0}
+               totalCountries={insight?.countries?.length || 0}
+               totalPages={insight?.pages?.length || 0}
+               />
+            )}
+
+            <div className='domkeywordsTable domkeywordsTable--sckeywords styled-scrollbar w-full overflow-auto min-h-[60vh]'>
+               <div className=' lg:min-w-[800px]'>
+                  {renderTableHeader()}
+                  <div className='domKeywords_keywords border-gray-200 min-h-[55vh] relative'>
+                     {['keywords', 'pages', 'countries', 'stats'].includes(activeTab) && insight && insightItems
+                           && (activeTab === 'stats' ? [...insightItems].reverse() : sortInsightItems(insightItems)).map(
+                              (item:SCInsightItem, index: number) => {
+                              const insightItemCount = insight ? insightItems : [];
+                              const lastItem = !!(insightItemCount && (index === insightItemCount.length));
+                              return <InsightItem key={index} item={item} type={activeTab} lastItem={lastItem} domain={domain?.domain || ''} />;
+                           },
+                        )
+                     }
+                     {isConsoleIntegrated && isLoading && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>Loading Insight...</p>
+                     )}
+                     {!isConsoleIntegrated && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>
+                        Google Search Console has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        </p>
+                     )}
+                  </div>
+               </div>
+            </div>
+         </div>
+         <Toaster position='bottom-center' containerClassName="react_toaster" />
+      </div>
+   );
+ };
+
+ export default SCInsight;

components/insight/InsightItem.tsx

@@ -0,0 +1,69 @@
+import React from 'react';
+import countries from '../../utils/countries';
+import Icon from '../common/Icon';
+import { formattedNum } from '../../utils/client/helpers';
+
+type InsightItemProps = {
+   item: SCInsightItem,
+   lastItem: boolean,
+   type: string,
+   domain: string
+}
+
+const InsightItem = ({ item, lastItem, type, domain }:InsightItemProps) => {
+   const { clicks, impressions, ctr, position, country = 'zzz', keyword, page, keywords = 0, countries: cntrs = 0, date } = item;
+   let firstItem = keyword;
+   if (type === 'pages') { firstItem = page; } if (type === 'stats') {
+      firstItem = date && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(date));
+   }
+   if (type === 'countries') { firstItem = countries[country] && countries[country][0]; }
+
+   return (
+      <div
+      className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
+      lg:flex lg:justify-between lg:items-center ${lastItem ? 'border-b-0' : ''}`}>
+
+         <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold'>
+            {type === 'countries' && <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} />}
+            {type === 'pages' && domain ? <a href={`https://${domain}${page}`} target='_blank' rel="noreferrer">{firstItem}</a> : firstItem}
+         </div>
+
+         <div className='keyword_pos text-center inline-block mr-3 lg:mr-0 lg:flex-1'>
+            <span className='mr-1 lg:hidden'>
+               <Icon type="tracking" size={14} color="#999" />
+            </span>
+            {Math.round(position)}
+         </div>
+
+         <div className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-14 p-2 text-base mt-[-55px] rounded right-5 lg:relative
+          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-40 lg:grow-0 lg:right-0 text-center font-semibold`}>
+            {formattedNum(clicks)}
+            <span className='block text-xs text-gray-500 lg:hidden'>Visits</span>
+         </div>
+
+         <div className='keyword_imp text-center inline-block mr-3 lg:mr-0 lg:flex-1'>
+            <span className='mr-1 lg:hidden'>
+               <Icon type="eye" size={14} color="#999" />
+            </span>
+            {formattedNum(impressions)}
+         </div>
+
+         <div className='keyword_ctr text-center inline-block mt-4 relative mr-3 lg:mr-0 lg:flex-1 lg:m-0 '>
+            <span className='mr-1 lg:hidden'>
+               <Icon type="target" size={14} color="#999" />
+            </span>
+            {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(ctr)}%
+         </div>
+
+         {(type === 'pages' || type === 'keywords') && (
+            <div className='keyword_imp text-center hidden lg:inline-block lg:flex-1'>{formattedNum(cntrs)}</div>
+         )}
+
+         {(type === 'countries' || type === 'pages') && (
+            <div className='keyword_imp text-center hidden lg:inline-block lg:flex-1'>{formattedNum(keywords)}</div>
+         )}
+      </div>
+   );
+};
+
+export default InsightItem;

components/insight/InsightStats.tsx

@@ -0,0 +1,122 @@
+import React, { useMemo } from 'react';
+import { Chart as ChartJS, CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend } from 'chart.js';
+import { Line } from 'react-chartjs-2';
+import { formattedNum } from '../../utils/client/helpers';
+
+ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend);
+
+type InsightStatsProps = {
+   stats: SearchAnalyticsStat[],
+   totalKeywords: number,
+   totalCountries: number,
+   totalPages: number,
+}
+
+const InsightStats = ({ stats = [], totalKeywords = 0, totalPages = 0 }:InsightStatsProps) => {
+    const totalStat = useMemo(() => {
+      const totals = stats.reduce((acc, item) => {
+          return {
+            impressions: item.impressions + acc.impressions,
+            clicks: item.clicks + acc.clicks,
+            position: item.position + acc.position,
+          };
+      }, { impressions: 0, clicks: 0, position: 0 });
+
+      return {
+          ...totals,
+          ctr: totals.impressions > 0 ? (totals.clicks / totals.impressions) * 100 : 0,
+      };
+    }, [stats]);
+
+   const chartData = useMemo(() => {
+      const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+      const chartSeries: {[key:string]: number[]} = { clicks: [], impressions: [], position: [], ctr: [] };
+      stats.forEach((item) => {
+         chartSeries.clicks.push(item.clicks);
+         chartSeries.impressions.push(item.impressions);
+         chartSeries.position.push(item.position);
+         chartSeries.ctr.push(item.ctr);
+      });
+      return {
+         labels: stats && stats.length > 0 ? stats.map((item) => `${new Date(item.date).getDate()}-${months[new Date(item.date).getMonth()]}`) : [],
+         series: chartSeries };
+   }, [stats]);
+
+   const renderChart = () => {
+      // Doc: https://www.chartjs.org/docs/latest/samples/line/multi-axis.html
+      const chartOptions = {
+         responsive: true,
+         maintainAspectRatio: false,
+         animation: false as const,
+         interaction: {
+            mode: 'index' as const,
+            intersect: false,
+          },
+         scales: {
+            x: {
+               grid: {
+                  drawOnChartArea: false,
+                },
+            },
+            y1: {
+               display: true,
+               position: 'right' as const,
+               grid: {
+                 drawOnChartArea: false,
+               },
+             },
+         },
+         plugins: {
+            legend: {
+                display: false,
+            },
+        },
+      };
+      const { clicks, impressions } = chartData.series || {};
+      const dataSet = [
+         { label: 'Visits', data: clicks, borderColor: 'rgb(117, 50, 205)', backgroundColor: 'rgba(117, 50, 205, 0.5)', yAxisID: 'y' },
+         { label: 'Impressions', data: impressions, borderColor: 'rgb(31, 205, 176)', backgroundColor: 'rgba(31, 205, 176, 0.5)', yAxisID: 'y1' },
+      ];
+      return <Line datasetIdKey={'xxx'} options={chartOptions} data={{ labels: chartData.labels, datasets: dataSet }} />;
+   };
+
+   return (
+      <div className='p-6 lg:border-t lg:border-gray-200'>
+         <div className=' flex font-bold flex-wrap lg:flex-nowrap'>
+            <div
+            className='flex-1 border border-gray-200 px-6 py-5 rounded mb-4 text-2xl text-violet-700 mr-5'
+            title={`${formattedNum(totalStat.clicks || 0)} Visits`}>
+               <span className=' block text-sm font-normal text-gray-500'>Visits</span>
+               {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(totalStat.clicks || 0).replace('T', 'K')}
+            </div>
+            <div
+            className='flex-1 border border-gray-200 px-6 py-5 rounded mb-4 text-2xl text-[#1fcdb0] lg:mr-5'
+            title={`${formattedNum(totalStat.impressions || 0)} Impressions`}>
+               <span className=' block text-sm font-normal text-gray-500'>Impressions</span>
+               {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(totalStat.impressions || 0).replace('T', 'K')}
+            </div>
+            <div className='flex-1 border border-gray-200 px-6 py-5 rounded mb-4 text-2xl text-gray-500 font-semibold mr-5'>
+               <span className=' block text-sm font-normal text-gray-500'>Avg Position</span>
+               {(totalStat.position ? Math.round(totalStat.position / stats.length) : 0)}
+            </div>
+            <div className='flex-1 border border-gray-200 px-6 py-5 rounded mb-4 text-2xl text-gray-500 font-semibold lg:mr-5'>
+               <span className=' block text-sm font-normal text-gray-500'>Avg CTR</span>
+               {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(totalStat.ctr || 0)}%
+            </div>
+            <div className='flex-1 border border-gray-200 px-6 py-5 rounded mb-4 text-2xl text-gray-500 font-semibold mr-5'>
+               <span className=' block text-sm font-normal text-gray-500'>Keywords</span>
+               {formattedNum(totalKeywords)}
+            </div>
+            <div className='flex-1 border border-gray-200 px-6 py-5 rounded mb-4 text-2xl text-gray-500 font-semibold'>
+               <span className=' block text-sm font-normal text-gray-500'>Pages</span>
+               {formattedNum(totalPages)}
+            </div>
+         </div>
+         <div className='h-80'>
+            {renderChart()}
+         </div>
+      </div>
+   );
+};
+
+export default InsightStats;

components/keywords/AddKeywords.tsx

@@ -1,4 +1,4 @@
-import React, { useState } from 'react';
+import React, { useCallback, useMemo, useRef, useState } from 'react';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
 import SelectField from '../common/SelectField';
@@ -7,6 +7,8 @@ import { useAddKeywords } from '../../services/keywords';
 
 type AddKeywordsProps = {
    keywords: KeywordType[],
+   scraperName: string,
+   allowsCity: boolean,
    closeModal: Function,
    domain: string
 }
@@ -17,24 +19,62 @@ type KeywordsInput = {
    country: string,
    domain: string,
    tags: string,
+   city?:string,
 }
 
-const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
+const AddKeywords = ({ closeModal, domain, keywords, scraperName = '', allowsCity = false }: AddKeywordsProps) => {
+   const inputRef = useRef(null);
+   const defCountry = localStorage.getItem('default_country') || 'US';
+
    const [error, setError] = useState<string>('');
-   const [newKeywordsData, setNewKeywordsData] = useState<KeywordsInput>({ keywords: '', device: 'desktop', country: 'US', domain, tags: '' });
+   const [showTagSuggestions, setShowTagSuggestions] = useState(false);
+   const [newKeywordsData, setNewKeywordsData] = useState<KeywordsInput>({ keywords: '', device: 'desktop', country: defCountry, domain, tags: '' });
    const { mutate: addMutate, isLoading: isAdding } = useAddKeywords(() => closeModal(false));
-   const deviceTabStyle = 'cursor-pointer px-3 py-2 rounded mr-2';
+
+   const existingTags: string[] = useMemo(() => {
+      const allTags = keywords.reduce((acc: string[], keyword) => [...acc, ...keyword.tags], []).filter((t) => t && t.trim() !== '');
+      return [...new Set(allTags)];
+   }, [keywords]);
+
+   const setDeviceType = useCallback((input:string) => {
+      let updatedDevice = '';
+      if (newKeywordsData.device.includes(input)) {
+         updatedDevice = newKeywordsData.device.replace(',', '').replace(input, '');
+      } else {
+         updatedDevice = newKeywordsData.device ? `${newKeywordsData.device},${input}` : input;
+      }
+      setNewKeywordsData({ ...newKeywordsData, device: updatedDevice });
+   }, [newKeywordsData]);
 
    const addKeywords = () => {
-      if (newKeywordsData.keywords) {
-         const keywordsArray = [...new Set(newKeywordsData.keywords.split('\n').map((item) => item.trim()).filter((item) => !!item))];
-         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}`);
-         const keywordExist = keywordsArray.filter((k) => currentKeywords.includes(`${k}-${newKeywordsData.device}-${newKeywordsData.country}`));
-         if (keywordExist.length > 0) {
+      const nkwrds = newKeywordsData;
+      if (nkwrds.keywords) {
+         const devices = nkwrds.device.split(',');
+         const multiDevice = nkwrds.device.includes(',') && devices.length > 1;
+         const keywordsArray = [...new Set(nkwrds.keywords.split('\n').map((item) => item.trim()).filter((item) => !!item))];
+         const currentKeywords = keywords.map((k) => `${k.keyword}-${k.device}-${k.country}${k.city ? `-${k.city}` : ''}`);
+
+         const keywordExist = keywordsArray.filter((k) =>
+            devices.some((device) => currentKeywords.includes(`${k}-${device}-${nkwrds.country}${nkwrds.city ? `-${nkwrds.city}` : ''}`)),
+         );
+
+         if (!multiDevice && (keywordsArray.length === 1 || currentKeywords.length === keywordExist.length) && keywordExist.length > 0) {
             setError(`Keywords ${keywordExist.join(',')} already Exist`);
             setTimeout(() => { setError(''); }, 3000);
          } else {
-            addMutate({ ...newKeywordsData, keywords: keywordsArray.join('\n') });
+            const newKeywords = keywordsArray.flatMap((k) =>
+               devices.filter((device) =>
+                 !currentKeywords.includes(`${k}-${device}-${nkwrds.country}${nkwrds.city ? `-${nkwrds.city}` : ''}`),
+               ).map((device) => ({
+                 keyword: k,
+                 device,
+                 country: nkwrds.country,
+                 domain: nkwrds.domain,
+                 tags: nkwrds.tags,
+                 city: nkwrds.city,
+               })),
+             );
+            addMutate(newKeywords);
          }
       } else {
          setError('Please Insert a Keyword');
@@ -42,6 +82,8 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
       }
    };
 
+   const deviceTabStyle = 'cursor-pointer px-2 py-2 rounded';
+
    return (
       <Modal closeModal={() => { closeModal(false); }} title={'Add New Keywords'} width="[420px]">
          <div data-testid="addkeywords_modal">
@@ -62,7 +104,10 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
                      selected={[newKeywordsData.country]}
                      options={Object.keys(countries).map((countryISO:string) => { return { label: countries[countryISO][0], value: countryISO }; })}
                      defaultLabel='All Countries'
-                     updateField={(updated:string[]) => setNewKeywordsData({ ...newKeywordsData, country: updated[0] })}
+                     updateField={(updated:string[]) => {
+                        setNewKeywordsData({ ...newKeywordsData, country: updated[0] });
+                        localStorage.setItem('default_country', updated[0]);
+                     }}
                      rounded='rounded'
                      maxHeight={48}
                      flags={true}
@@ -70,25 +115,63 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => {
                   </div>
                   <ul className='flex text-xs font-semibold text-gray-500'>
                      <li
-                        className={`${deviceTabStyle} ${newKeywordsData.device === 'desktop' ? '  bg-indigo-50 text-gray-700' : ''}`}
-                        onClick={() => setNewKeywordsData({ ...newKeywordsData, device: 'desktop' })}
-                        ><Icon type='desktop' classes={'top-[3px]'} size={15} /> <i className='not-italic hidden lg:inline-block'>Desktop</i></li>
+                        className={`${deviceTabStyle} mr-2 ${newKeywordsData.device.includes('desktop') ? '  bg-indigo-50 text-indigo-700' : ''}`}
+                        onClick={() => setDeviceType('desktop')}>
+                           <Icon type='desktop' classes={'top-[3px]'} size={15} /> <i className='not-italic hidden lg:inline-block'>Desktop</i>
+                           <Icon type='check' classes={'pl-1'} size={12} color={newKeywordsData.device.includes('desktop') ? '#4338ca' : '#bbb'} />
+                        </li>
                      <li
-                        className={`${deviceTabStyle} ${newKeywordsData.device === 'mobile' ? '  bg-indigo-50 text-gray-700' : ''}`}
-                        onClick={() => setNewKeywordsData({ ...newKeywordsData, device: 'mobile' })}
-                        ><Icon type='mobile' /> <i className='not-italic hidden lg:inline-block'>Mobile</i></li>
+                        className={`${deviceTabStyle} ${newKeywordsData.device.includes('mobile') ? '  bg-indigo-50 text-indigo-700' : ''}`}
+                        onClick={() => setDeviceType('mobile')}>
+                           <Icon type='mobile' /> <i className='not-italic hidden lg:inline-block'>Mobile</i>
+                           <Icon type='check' classes={'pl-1'} size={12} color={newKeywordsData.device.includes('mobile') ? '#4338ca' : '#bbb'} />
+                        </li>
                   </ul>
                </div>
-
                <div className='relative'>
-                  {/* TODO:  Insert Existing Tags as Suggestions */}
                   <input
-                     className='w-full border rounded border-gray-200 py-2 px-4 pl-8 outline-none focus:border-indigo-300'
-                     placeholder='Insert Tags'
+                     className='w-full border rounded border-gray-200 py-2 px-4 pl-12 outline-none focus:border-indigo-300'
+                     placeholder='Insert Tags (Optional)'
                      value={newKeywordsData.tags}
                      onChange={(e) => setNewKeywordsData({ ...newKeywordsData, tags: e.target.value })}
                   />
-                  <span className='absolute text-gray-400 top-2 left-2'><Icon type="tags" size={16} /></span>
+                  <span className='absolute text-gray-400 top-3 left-2 cursor-pointer' onClick={() => setShowTagSuggestions(!showTagSuggestions)}>
+                     <Icon type="tags" size={16} color={showTagSuggestions ? '#777' : '#aaa'} />
+                     <Icon type={showTagSuggestions ? 'caret-up' : 'caret-down'} size={14} color={showTagSuggestions ? '#666' : '#aaa'} />
+                  </span>
+                  {showTagSuggestions && (
+                     <ul className={`absolute z-50
+                     bg-white border border-t-0 border-gray-200 rounded rounded-t-none w-full`}>
+                        {existingTags.length > 0 && existingTags.map((tag, index) => {
+                           return newKeywordsData.tags.split(',').map((t) => t.trim()).includes(tag) === false && <li
+                                    className=' p-2 cursor-pointer hover:text-indigo-600 hover:bg-indigo-50 transition'
+                                    key={index}
+                                    onClick={() => {
+                                       const tagInput = newKeywordsData.tags;
+                                       // eslint-disable-next-line no-nested-ternary
+                                       const tagToInsert = tagInput + (tagInput.trim().slice(-1) === ',' ? '' : (tagInput.trim() ? ', ' : '')) + tag;
+                                       setNewKeywordsData({ ...newKeywordsData, tags: tagToInsert });
+                                       setShowTagSuggestions(false);
+                                       if (inputRef?.current) (inputRef.current as HTMLInputElement).focus();
+                                    }}>
+                                       <Icon type='tags' size={14} color='#bbb' /> {tag}
+                                    </li>;
+                        })}
+                        {existingTags.length === 0 && <p>No Existing Tags Found... </p>}
+                     </ul>
+                  )}
+               </div>
+               <div className='relative mt-2'>
+                  <input
+                     className={`w-full border rounded border-gray-200 py-2 px-4 pl-8 
+                     outline-none focus:border-indigo-300 ${!allowsCity ? ' cursor-not-allowed' : ''} `}
+                     disabled={!allowsCity}
+                     title={!allowsCity ? `Your scraper ${scraperName} doesn't have city level scraping feature.` : ''}
+                     placeholder={`City (Optional)${!allowsCity ? `. Not avaialable for ${scraperName}.` : ''}`}
+                     value={newKeywordsData.city}
+                     onChange={(e) => setNewKeywordsData({ ...newKeywordsData, city: e.target.value })}
+                  />
+                  <span className='absolute text-gray-400 top-2 left-2'><Icon type="city" size={16} /></span>
                </div>
             </div>
             {error && <div className='w-full mt-4 p-3 text-sm bg-red-50 text-red-700'>{error}</div>}

components/keywords/AddTags.tsx

@@ -0,0 +1,88 @@
+import { useRef, useState } from 'react';
+import { useUpdateKeywordTags } from '../../services/keywords';
+import Icon from '../common/Icon';
+import Modal from '../common/Modal';
+
+type AddTagsProps = {
+   keywords: KeywordType[],
+   existingTags: string[],
+   closeModal: Function
+}
+
+const AddTags = ({ keywords = [], existingTags = [], closeModal }: AddTagsProps) => {
+   const [tagInput, setTagInput] = useState(() => (keywords.length === 1 ? keywords[0].tags.join(', ') : ''));
+   const [inputError, setInputError] = useState('');
+   const [showSuggestions, setShowSuggestions] = useState(false);
+   const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
+   const inputRef = useRef(null);
+
+   const addTag = () => {
+      if (keywords.length === 0) { return; }
+      if (!tagInput && keywords.length > 1) {
+         setInputError('Please Insert a Tag!');
+         setTimeout(() => { setInputError(''); }, 3000);
+         return;
+      }
+
+      const tagsArray = tagInput.split(',').map((t) => t.trim());
+      const tagsPayload:any = {};
+      keywords.forEach((keyword:KeywordType) => {
+         tagsPayload[keyword.ID] = keywords.length === 1 ? tagsArray : [...(new Set([...keyword.tags, ...tagsArray]))];
+      });
+      updateMutate({ tags: tagsPayload });
+   };
+
+   return (
+      <Modal closeModal={() => { closeModal(false); }} title={`Add New Tags to ${keywords.length} Selected Keyword`}>
+         <div className="relative">
+            {inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
+            <span className='absolute text-gray-400 top-3 left-2 cursor-pointer' onClick={() => setShowSuggestions(!showSuggestions)}>
+               <Icon type="tags" size={16} color={showSuggestions ? '#777' : '#aaa'} />
+               <Icon type={showSuggestions ? 'caret-up' : 'caret-down'} size={14} color={showSuggestions ? '#666' : '#aaa'} />
+            </span>
+            <input
+               ref={inputRef}
+               className='w-full border rounded border-gray-200 py-3 px-4 pl-12 outline-none focus:border-indigo-300'
+               placeholder='Insert Tags. eg: tag1, tag2'
+               value={tagInput}
+               onChange={(e) => setTagInput(e.target.value)}
+               onKeyDown={(e) => {
+                  if (e.code === 'Enter') {
+                    e.preventDefault();
+                    addTag();
+                  }
+               }}
+            />
+            {showSuggestions && (
+               <ul className={`absolute z-50
+               bg-white border border-t-0 border-gray-200 rounded rounded-t-none w-full`}>
+                  {existingTags.length > 0 && existingTags.map((tag, index) => {
+                     return tagInput.split(',').map((t) => t.trim()).includes(tag) === false && <li
+                              className=' p-2 cursor-pointer hover:text-indigo-600 hover:bg-indigo-50 transition'
+                              key={index}
+                              onClick={() => {
+                                 // eslint-disable-next-line no-nested-ternary
+                                 const tagToInsert = tagInput + (tagInput.trim().slice(-1) === ',' ? '' : (tagInput.trim() ? ', ' : '')) + tag;
+                                 setTagInput(tagToInsert);
+                                 setShowSuggestions(false);
+                                 if (inputRef?.current) (inputRef.current as HTMLInputElement).focus();
+                              }}>
+                                 <Icon type='tags' size={14} color='#bbb' /> {tag}
+                              </li>;
+                  })}
+                  {existingTags.length === 0 && <p>No Existing Tags Found... </p>}
+               </ul>
+            )}
+
+            <button
+            className=" absolute right-2 top-2 cursor-pointer rounded p-2 px-4 bg-indigo-600 text-white font-semibold text-sm"
+            onClick={addTag}>
+               Apply
+            </button>
+         </div>
+      </Modal>
+
+   );
+};
+
+export default AddTags;

components/keywords/Keyword.tsx

@@ -4,27 +4,53 @@ import dayjs from 'dayjs';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
 import ChartSlim from '../common/ChartSlim';
-import { generateTheChartData } from '../common/generateChartData';
+import KeywordPosition from './KeywordPosition';
+import { generateTheChartData } from '../../utils/client/generateChartData';
+import { formattedNum } from '../../utils/client/helpers';
 
 type KeywordProps = {
    keywordData: KeywordType,
    selected: boolean,
+   index: number,
    refreshkeyword: Function,
    favoriteKeyword: Function,
    removeKeyword: Function,
    selectKeyword: Function,
    manageTags: Function,
    showKeywordDetails: Function,
-   lastItem?:boolean
+   lastItem?:boolean,
+   showSCData: boolean,
+   scDataType: string,
+   style: Object,
+   maxTitleColumnWidth: number,
+   tableColumns? : string[]
 }
 
 const Keyword = (props: KeywordProps) => {
-   const { keywordData, refreshkeyword, favoriteKeyword, removeKeyword, selectKeyword, selected, showKeywordDetails, manageTags, lastItem } = props;
    const {
-      keyword, domain, ID, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
+      keywordData,
+      refreshkeyword,
+      favoriteKeyword,
+      removeKeyword,
+      selectKeyword,
+      selected,
+      showKeywordDetails,
+      manageTags,
+      lastItem,
+      showSCData = true,
+      style,
+      index,
+      scDataType = 'threeDays',
+      tableColumns = [],
+      maxTitleColumnWidth,
+   } = props;
+   const {
+      keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false, volume,
    } = keywordData;
+
    const [showOptions, setShowOptions] = useState(false);
    const [showPositionError, setPositionError] = useState(false);
+
    const turncatedURL = useMemo(() => {
       return url.replace(`https://${domain}`, '').replace(`https://www.${domain}`, '').replace(`http://${domain}`, '');
    }, [url, domain]);
@@ -42,28 +68,36 @@ const Keyword = (props: KeywordProps) => {
          const historySorted = historyArray.sort((a, b) => a.date - b.date);
          const previousPos = historySorted[historySorted.length - 2].position;
          status = previousPos === 0 ? position : previousPos - position;
+         if (position === 0 && previousPos > 0) {
+            status = previousPos - 100;
+         }
       }
       return status;
    }, [history, position]);
 
-   const optionsButtonStyle = 'block px-2 py-2 cursor-pointer hover:bg-indigo-50 hover:text-blue-700';
+   const bestPosition: false | {position: number, date: string} = useMemo(() => {
+      let bestPos;
+      if (Object.keys(history).length > 0) {
+         const historyArray = Object.keys(history).map((itemID) => ({ date: itemID, position: history[itemID] }))
+             .sort((a, b) => a.position - b.position).filter((el) => (el.position > 0));
+         if (historyArray[0]) {
+            bestPos = { ...historyArray[0] };
+         }
+      }
 
-   const renderPosition = () => {
-      if (position === 0) {
-         return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
-      }
-      if (updating) {
-         return <span title='Updating Keyword Position'><Icon type="loading" /></span>;
-      }
-      return position;
-   };
+      return bestPos || false;
+   }, [history]);
+
+   const optionsButtonStyle = 'block px-2 py-2 cursor-pointer hover:bg-indigo-50 hover:text-blue-700';
 
    return (
       <div
-      key={keyword}
+      key={keyword + ID}
+      style={style}
       className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
       lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
-         <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold cursor-pointer'>
+
+         <div className=' w-3/4 font-semibold cursor-pointer lg:flex-1 lg:shrink-0 lg:basis-28 lg:w-auto lg:flex lg:items-center'>
             <button
                className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
                ${selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
@@ -72,9 +106,15 @@ const Keyword = (props: KeywordProps) => {
                   <Icon type="check" size={10} />
             </button>
             <a
-            className='py-2 hover:text-blue-600'
-            onClick={() => showKeywordDetails()}>
-               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />{keyword}
+            style={{ maxWidth: `${maxTitleColumnWidth - 35}px` }}
+            className={'py-2 hover:text-blue-600 lg:flex lg:items-center w-full'}
+            onClick={() => showKeywordDetails()}
+            title={keyword}
+            >
+               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />
+               <span className='inline-block text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-50px)]'>
+                  {keyword}{city ? ` (${city})` : ''}
+               </span>
             </a>
             {sticky && <button className='ml-2 relative top-[2px]' title='Favorite'><Icon type="star-filled" size={16} color="#fbd346" /></button>}
             {lastUpdateError && lastUpdateError.date
@@ -83,28 +123,77 @@ const Keyword = (props: KeywordProps) => {
                </button>
             }
          </div>
+
          <div
          className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative
-          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-40 lg:grow-0 lg:right-0 text-center font-semibold`}>
-            {renderPosition()}
+          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center font-semibold`}>
+            <KeywordPosition position={position} updating={updating} />
             {!updating && positionChange > 0 && <i className=' not-italic ml-1 text-xs text-[#5ed7c3]'>▲ {positionChange}</i>}
             {!updating && positionChange < 0 && <i className=' not-italic ml-1 text-xs text-red-300'>▼ {positionChange}</i>}
          </div>
+
+         <div
+         title={bestPosition && bestPosition.date
+            ? new Date(bestPosition.date).toLocaleDateString('en-US', { weekday: 'long', year: 'numeric', month: 'short', day: 'numeric' }) : ''
+         }
+         className={`keyword_best mr-1 bg-[#f8f9ff] float-right mt-8 w-14 rounded right-5 lg:relative lg:block
+          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:mr-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-16 lg:grow-0 lg:right-0 text-center font-semibold
+          ${!tableColumns.includes('Best') ? 'lg:hidden' : ''}
+          `}>
+            {bestPosition ? bestPosition.position || '-' : (position || '-')}
+         </div>
+
          {chartData.labels.length > 0 && (
-            <div className='lg:flex-1 hidden lg:block'>
+            <div
+               className={`hidden basis-20 grow-0 cursor-pointer lg:block ${!tableColumns.includes('History') ? 'lg:hidden' : ''}`}
+               onClick={() => showKeywordDetails()}>
                <ChartSlim labels={chartData.labels} sreies={chartData.sreies} />
             </div>
          )}
+
+         <div
+         className={`hidden bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative lg:block
+          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center
+          ${!tableColumns.includes('Volume') ? 'lg:hidden' : ''}
+          `}>
+            {formattedNum(volume)}
+         </div>
+
          <div
          className={`keyword_url inline-block mt-4 mr-5 ml-5 lg:flex-1 text-gray-400 lg:m-0 max-w-[70px] 
-         overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5`}>
-            <span className='mr-3 lg:hidden'><Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}</div>
+         overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5 lg:pl-3`}>
+            <a href={url} target="_blank" rel="noreferrer"><span className='mr-3 lg:hidden'>
+               <Icon type="link-alt" size={14} color="#999" /></span>{turncatedURL || '-'}
+            </a>
+         </div>
+
          <div
-         className='inline-block mt-[4] top-[-5px] relative lg:flex-1 lg:m-0'>
+         className='inline-block mt-[4] top-[-5px] relative lg:flex-1 lg:m-0 lg:top-0 max-w-[150px]'>
             <span className='mr-2 lg:hidden'><Icon type="clock" size={14} color="#999" /></span>
             <TimeAgo title={dayjs(lastUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={lastUpdated} />
          </div>
-         <div className='absolute right-7 mt-[-10px] lg:flex-1 lg:basis-5 lg:grow-0 lg:shrink-0 lg:relative lg:mt-0 lg:right-auto'>
+
+         {showSCData && tableColumns.includes('Search Console') && (
+            <div className='keyword_sc_data min-w-[170px] lg:max-w-[170px] text-xs mt-4 pt-2 border-t border-gray-100 top-[6px]
+            relative flex justify-between text-center lg:flex-1 lg:text-sm lg:m-0 lg:mt-0 lg:border-t-0 lg:pt-0 lg:top-0'>
+               <span className='min-w-[40px]'>
+                  <span className='lg:hidden'>SC Position: </span>
+                  <KeywordPosition
+                  position={keywordData?.scData?.position[scDataType as keyof KeywordSCDataChild] || 0}
+                  type='sc'
+                  />
+               </span>
+               <span className='min-w-[40px]'>
+                  <span className='lg:hidden'>Impressions: </span>{keywordData?.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0}
+               </span>
+               <span className='min-w-[40px]'>
+                  <span className='lg:hidden'>Visits: </span>{keywordData?.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0}
+               </span>
+               {/* <span>{keywordData?.scData?.ctr[scDataType] || '0.00%'}</span> */}
+            </div>
+         )}
+
+         <div className='absolute right-4 mt-[-10px] top-2 lg:flex-1 lg:basis-5 lg:grow-0 lg:shrink-0 lg:relative lg:right-[-10px]'>
             <button
             className={`keyword_dots rounded px-1 text-indigo-300 hover:bg-indigo-50 ${showOptions ? 'bg-indigo-50 text-indigo-600 ' : ''}`}
             onClick={() => setShowOptions(!showOptions)}>
@@ -133,8 +222,10 @@ const Keyword = (props: KeywordProps) => {
                </ul>
             )}
          </div>
+
          {lastUpdateError && lastUpdateError.date && showPositionError && (
-            <div className=' absolute mt-[-70px] p-2 bg-white z-30 border border-red-200 rounded w-[220px] left-4 shadow-sm text-xs lg:bottom-12'>
+            <div className={`absolute p-2 bg-white z-30 border border-red-200 rounded w-[220px] left-4 shadow-sm text-xs 
+            ${index > 2 ? 'lg:bottom-12 mt-[-70px]' : ' top-12'}`}>
                Error Updating Keyword position (Tried <TimeAgo
                                                          title={dayjs(lastUpdateError.date).format('DD-MMM-YYYY, hh:mm:ss A')}
                                                          date={lastUpdateError.date} />)
@@ -145,7 +236,8 @@ const Keyword = (props: KeywordProps) => {
                   {lastUpdateError.scraper && <strong className='capitalize'>{lastUpdateError.scraper}: </strong>}{lastUpdateError.error}
                </div>
             </div>
-            )}
+         )}
+
       </div>
    );
  };

components/keywords/KeywordDetails.tsx

@@ -1,10 +1,12 @@
-import React, { useEffect, useMemo, useRef, useState } from 'react';
+import React, { useLayoutEffect, useMemo, useRef, useState } from 'react';
 import dayjs from 'dayjs';
 import Icon from '../common/Icon';
 import countries from '../../utils/countries';
 import Chart from '../common/Chart';
 import SelectField from '../common/SelectField';
-import { generateTheChartData } from '../common/generateChartData';
+import { useFetchSingleKeyword } from '../../services/keywords';
+import useOnKey from '../../hooks/useOnKey';
+import { generateTheChartData } from '../../utils/client/generateChartData';
 
 type KeywordDetailsProps = {
    keyword: KeywordType,
@@ -13,11 +15,12 @@ type KeywordDetailsProps = {
 
 const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
    const updatedDate = new Date(keyword.lastUpdated);
-   const [keywordHistory, setKeywordHistory] = useState<KeywordHistory>(keyword.history);
-   const [keywordSearchResult, setKeywordSearchResult] = useState<KeywordLastResult[]>([]);
    const [chartTime, setChartTime] = useState<string>('30');
    const searchResultContainer = useRef<HTMLDivElement>(null);
    const searchResultFound = useRef<HTMLDivElement>(null);
+   const { data: keywordData } = useFetchSingleKeyword(keyword.ID);
+   const keywordHistory: KeywordHistory = keywordData?.history || keyword.history;
+   const keywordSearchResult: KeywordLastResult = keywordData?.searchResult || keyword.history;
    const dateOptions = [
       { label: 'Last 7 Days', value: '7' },
       { label: 'Last 30 Days', value: '30' },
@@ -26,39 +29,9 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
       { label: 'All Time', value: 'all' },
    ];
 
-   useEffect(() => {
-      const fetchFullKeyword = async () => {
-         try {
-            const fetchURL = `${window.location.origin}/api/keyword?id=${keyword.ID}`;
-            const res = await fetch(fetchURL, { method: 'GET' }).then((result) => result.json());
-            if (res.keyword) {
-               console.log(res.keyword, new Date().getTime());
-               setKeywordHistory(res.keyword.history || []);
-               setKeywordSearchResult(res.keyword.lastResult || []);
-            }
-         } catch (error) {
-            console.log(error);
-         }
-      };
-      if (keyword.lastResult.length === 0) {
-         fetchFullKeyword();
-      }
-   }, [keyword]);
+   useOnKey('Escape', closeDetails);
 
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            console.log(event.key);
-            closeDetails();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeDetails]);
-
-   useEffect(() => {
+   useLayoutEffect(() => {
       if (keyword.position < 100 && keyword.position > 0 && searchResultFound?.current) {
          searchResultFound.current.scrollIntoView({
             behavior: 'smooth',
@@ -79,13 +52,16 @@ const KeywordDetails = ({ keyword, closeDetails }:KeywordDetailsProps) => {
    };
 
    return (
-       <div className="keywordDetails fixed w-full h-screen top-0 left-0 z-30" onClick={closeOnBGClick} data-testid="keywordDetails">
+       <div className="keywordDetails fixed w-full h-screen top-0 left-0 z-[99999]" onClick={closeOnBGClick} data-testid="keywordDetails">
             <div className="keywordDetails absolute w-full lg:w-5/12 bg-white customShadow top-0 right-0 h-screen" >
-               <div className='keywordDetails__header p-6 border-b border-b-slate-200 text-slate-500'>
+               <div className='keywordDetails__header p-6 border-b border-b-slate-200 text-slate-600'>
                   <h3 className=' text-lg font-bold'>
                      <span title={countries[keyword.country][0]}
                      className={`fflag fflag-${keyword.country} w-[18px] h-[12px] mr-2`} /> {keyword.keyword}
-                     <span className='py-1 px-2 rounded bg-blue-50 text-blue-700 text-xs font-bold'>{keyword.position}</span>
+                     <span
+                     className={`py-1 px-2 ml-2 rounded bg-blue-50 ${keyword.position === 0 ? 'text-gray-500' : 'text-blue-700'}  text-xs font-bold`}>
+                        {keyword.position === 0 ? 'Not in First 100' : keyword.position}
+                     </span>
                   </h3>
                   <button
                   className='absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'

components/keywords/KeywordFilter.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useEffect, useMemo } from 'react';
+import React, { useState, useMemo } from 'react';
 import Icon from '../common/Icon';
 import SelectField, { SelectionOption } from '../common/SelectField';
 import countries from '../../utils/countries';
@@ -9,14 +9,14 @@ type KeywordFilterProps = {
    setDevice: Function,
    filterParams: KeywordFilters,
    filterKeywords: Function,
-   keywords: KeywordType[],
+   keywords: KeywordType[] | SearchAnalyticsItem[],
    updateSort: Function,
-   sortBy: string
-}
-
-type KeywordCountState = {
-   desktop: number,
-   mobile: number
+   sortBy: string,
+   integratedConsole?: boolean,
+   isConsole?: boolean,
+   SCcountries?: string[];
+   updateColumns?: Function,
+   tableColumns?: string[]
 }
 
 const KeywordFilters = (props: KeywordFilterProps) => {
@@ -25,24 +25,32 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       setDevice,
       filterKeywords,
       allTags = [],
-      keywords,
+      keywords = [],
       updateSort,
       sortBy,
-      filterParams } = props;
-   const [keywordCounts, setKeywordCounts] = useState<KeywordCountState>({ desktop: 0, mobile: 0 });
+      filterParams,
+      isConsole = false,
+      integratedConsole = false,
+      updateColumns,
+      SCcountries = [],
+      tableColumns = [],
+    } = props;
    const [sortOptions, showSortOptions] = useState(false);
    const [filterOptions, showFilterOptions] = useState(false);
+   const [columnOptions, showColumnOptions] = useState(false);
 
-   useEffect(() => {
-      const keyWordCount = { desktop: 0, mobile: 0 };
-      keywords.forEach((k) => {
-         if (k.device === 'desktop') {
-            keyWordCount.desktop += 1;
-         } else {
-            keyWordCount.mobile += 1;
-         }
-      });
-      setKeywordCounts(keyWordCount);
+   const keywordCounts = useMemo(() => {
+      const counts = { desktop: 0, mobile: 0 };
+      if (keywords && keywords.length > 0) {
+         keywords.forEach((k) => {
+            if (k.device === 'desktop') {
+               counts.desktop += 1;
+            } else {
+               counts.mobile += 1;
+            }
+         });
+      }
+      return counts;
    }, [keywords]);
 
    const filterCountry = (cntrs:string[]) => filterKeywords({ ...filterParams, countries: cntrs });
@@ -55,12 +63,24 @@ const KeywordFilters = (props: KeywordFilterProps) => {
    };
 
    const countryOptions = useMemo(() => {
-      const optionObject = Object.keys(countries).map((countryISO:string) => ({
-         label: countries[countryISO][0],
-         value: countryISO,
-      }));
+      const optionObject:{label:string, value:string}[] = [];
+
+      if (!isConsole) {
+         const allCountries = Array.from(keywords as KeywordType[])
+         .map((keyword) => keyword.country)
+         .reduce<string[]>((acc, country) => [...acc, country], [])
+         .filter((t) => t && t.trim() !== '');
+         [...new Set(allCountries)].forEach((c) => optionObject.push({ label: countries[c][0], value: c }));
+      } else {
+         Object.keys(countries).forEach((countryISO:string) => {
+            if ((SCcountries.includes(countryISO))) {
+               optionObject.push({ label: countries[countryISO][0], value: countryISO });
+            }
+         });
+      }
+
       return optionObject;
-   }, []);
+   }, [SCcountries, isConsole, keywords]);
 
    const sortOptionChoices: SelectionOption[] = [
       { value: 'pos_asc', label: 'Top Position' },
@@ -69,7 +89,31 @@ const KeywordFilters = (props: KeywordFilterProps) => {
       { value: 'date_desc', label: 'Oldest' },
       { value: 'alpha_asc', label: 'Alphabetically(A-Z)' },
       { value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
+      { value: 'vol_asc', label: 'Lowest Search Volume' },
+      { value: 'vol_desc', label: 'Highest Search Volume' },
    ];
+
+   const columnOptionChoices: {label: string, value: string, locked: boolean}[] = [
+      { value: 'Keyword', label: 'Keyword', locked: true },
+      { value: 'Position', label: 'Position', locked: true },
+      { value: 'URL', label: 'URL', locked: true },
+      { value: 'Updated', label: 'Updated', locked: true },
+      { value: 'Best', label: 'Best', locked: false },
+      { value: 'History', label: 'History', locked: false },
+      { value: 'Volume', label: 'Volume', locked: false },
+      { value: 'Search Console', label: 'Search Console', locked: false },
+   ];
+   if (integratedConsole) {
+      sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
+      sortOptionChoices.push({ value: 'imp_asc', label: 'Least Viewed' });
+      sortOptionChoices.push({ value: 'visits_desc', label: 'Most Visited' });
+      sortOptionChoices.push({ value: 'visits_asc', label: 'Least Visited' });
+   }
+   if (isConsole) {
+      sortOptionChoices.splice(2, 2);
+      sortOptionChoices.push({ value: 'ctr_asc', label: 'Highest CTR' });
+      sortOptionChoices.push({ value: 'ctr_desc', label: 'Lowest CTR' });
+   }
    const sortItemStyle = (sortType:string) => {
       return `cursor-pointer py-2 px-3 hover:bg-[#FCFCFF] ${sortBy === sortType ? 'bg-indigo-50 text-indigo-600 hover:bg-indigo-50' : ''}`;
    };
@@ -119,15 +163,17 @@ const KeywordFilters = (props: KeywordFilterProps) => {
                      flags={true}
                   />
                </div>
-               <div className={'tags_filter mb-2 lg:mb-0'}>
-                  <SelectField
-                     selected={filterParams.tags}
-                     options={allTags.map((tag:string) => ({ label: tag, value: tag }))}
-                     defaultLabel='All Tags'
-                     updateField={(updated:string[]) => filterTags(updated)}
-                     emptyMsg="No Tags Found for this Domain"
-                  />
-               </div>
+               {!isConsole && (
+                  <div className={'tags_filter mb-2 lg:mb-0'}>
+                     <SelectField
+                        selected={filterParams.tags}
+                        options={allTags.map((tag:string) => ({ label: tag, value: tag }))}
+                        defaultLabel='All Tags'
+                        updateField={(updated:string[]) => filterTags(updated)}
+                        emptyMsg="No Tags Found for this Domain"
+                     />
+                  </div>
+               )}
                <div className={'mb-2 lg:mb-0'}>
                   <input
                      data-testid="filter_input"
@@ -150,7 +196,8 @@ const KeywordFilters = (props: KeywordFilterProps) => {
                   {sortOptions && (
                      <ul
                      data-testid="sort_options"
-                     className='sort_options mt-2 border absolute min-w-[0] right-0 rounded-lg max-h-96 bg-white z-50 w-44'>
+                     className='sort_options mt-2 border absolute w-48 min-w-[0] right-0 rounded-lg
+                     max-h-96 bg-white z-[9999] overflow-y-auto styled-scrollbar'>
                         {sortOptionChoices.map((sortOption) => {
                            return <li
                                     key={sortOption.value}
@@ -162,6 +209,43 @@ const KeywordFilters = (props: KeywordFilterProps) => {
                      </ul>
                   )}
                </div>
+               {!isConsole && (
+                  <div className='relative'>
+                  <button
+                  data-testid="columns_button"
+                  className={`px-2 py-1 rounded ${columnOptions ? ' bg-indigo-100 text-blue-700' : ''}`}
+                  title='Show/Hide Columns'
+                  onClick={() => showColumnOptions(!columnOptions)}
+                  >
+                     <Icon type='eye-closed' size={18} />
+                  </button>
+                  {columnOptions && (
+                     <ul
+                     data-testid="sort_options"
+                     className='sort_options mt-2 border absolute w-48 min-w-[0] right-0 rounded-lg
+                     max-h-96 bg-white z-[9999] overflow-y-auto styled-scrollbar border-gray-200 '>
+                        {columnOptionChoices.map(({ value, label, locked }) => {
+                           return <li
+                                    key={value}
+                                    className={sortItemStyle(value) + (locked ? 'bg-gray-50 cursor-not-allowed pointer-events-none' : '') }
+                                    onClick={() => { if (updateColumns) { updateColumns(value); } showColumnOptions(false); }}
+                                    >
+                                       <span className={' inline-block px-[3px] border border-gray-200  rounded-[4px] w-5'}>
+                                          <Icon
+                                          title={locked ? 'Cannot be Hidden' : ''}
+                                          type={locked ? 'lock' : 'check'}
+                                          color={!tableColumns.includes(value) && !locked ? 'transparent' : '#999' }
+                                          size={12}
+                                          />
+                                        </span>
+                                       {' '}{label}
+
+                                    </li>;
+                        })}
+                     </ul>
+                  )}
+               </div>
+               )}
          </div>
       </div>
    );

components/keywords/KeywordPosition.tsx

@@ -0,0 +1,19 @@
+import Icon from '../common/Icon';
+
+type KeywordPositionProps = {
+   position: number,
+   updating?: boolean,
+   type?: string,
+}
+
+const KeywordPosition = ({ position = 0, type = '', updating = false }:KeywordPositionProps) => {
+   if (!updating && position === 0) {
+      return <span className='text-gray-400' title='Not in Top 100'>{'>100'}</span>;
+   }
+   if (updating && type !== 'sc') {
+      return <span title='Updating Keyword Position'><Icon type="loading" /></span>;
+   }
+   return <>{Math.round(position)}</>;
+};
+
+export default KeywordPosition;

components/keywords/KeywordTagManager.tsx

@@ -2,6 +2,7 @@ import { useState } from 'react';
 import { useUpdateKeywordTags } from '../../services/keywords';
 import Icon from '../common/Icon';
 import Modal from '../common/Modal';
+import AddTags from './AddTags';
 
 type keywordTagManagerProps = {
    keyword: KeywordType|undefined,
@@ -9,10 +10,9 @@ type keywordTagManagerProps = {
    allTags: string[]
 }
 
-const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
-   const [tagInput, setTagInput] = useState('');
-   const [inputError, setInputError] = useState('');
-   const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); });
+const KeywordTagManager = ({ keyword, allTags = [], closeModal }: keywordTagManagerProps) => {
+   const [showAddTag, setShowAddTag] = useState<boolean>(false);
+   const { mutate: updateMutate } = useUpdateKeywordTags(() => { });
 
    const removeTag = (tag:String) => {
       if (!keyword) { return; }
@@ -20,24 +20,6 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
       updateMutate({ tags: { [keyword.ID]: newTags } });
    };
 
-   const addTag = () => {
-      if (!keyword) { return; }
-      if (!tagInput) {
-         setInputError('Please Insert a Tag!');
-         setTimeout(() => { setInputError(''); }, 3000);
-         return;
-      }
-      if (keyword.tags.includes(tagInput)) {
-         setInputError('Tag Exist!');
-         setTimeout(() => { setInputError(''); }, 3000);
-         return;
-      }
-
-      console.log('New Tag: ', tagInput);
-      const newTags = [...keyword.tags, tagInput.trim()];
-      updateMutate({ tags: { [keyword.ID]: newTags } });
-   };
-
    return (
       <Modal closeModal={() => { closeModal(false); }} title={`Tags for Keyword "${keyword && keyword.keyword}"`}>
          <div className="text-sm my-8 ">
@@ -53,31 +35,28 @@ const KeywordTagManager = ({ keyword, closeModal }: keywordTagManagerProps) => {
                                  </button>
                            </li>;
                   })}
+                  <li className='inline-block py-1 px-1'>
+                     <button
+                     title='Add New Tag'
+                     className="cursor-pointer rounded p-1 px-3 bg-indigo-600 text-white font-semibold text-sm"
+                     onClick={() => setShowAddTag(true)}>+</button>
+                  </li>
                </ul>
             )}
             {keyword && keyword.tags.length === 0 && (
-               <div className="text-center w-full text-gray-500">No Tags Added to this Keyword.</div>
+               <div className="text-center w-full text-gray-500">
+                  No Tags Added to this Keyword. <button className=' text-indigo-600' onClick={() => setShowAddTag(true)}>+ Add Tag</button>
+               </div>
             )}
          </div>
-         <div className="relative">
-            {inputError && <span className="absolute top-[-24px] text-red-400 text-sm font-semibold">{inputError}</span>}
-            <span className='absolute text-gray-400 top-3 left-2'><Icon type="tags" size={16} /></span>
-            <input
-               className='w-full border rounded border-gray-200 py-3 px-4 pl-8 outline-none focus:border-indigo-300'
-               placeholder='Insert Tags'
-               value={tagInput}
-               onChange={(e) => setTagInput(e.target.value)}
-               onKeyDown={(e) => {
-                  if (e.code === 'Enter') {
-                    e.preventDefault();
-                    addTag();
-                  }
-               }}
-            />
-            <button className=" absolute right-2 top-2 cursor-pointer rounded p-1 px-4 bg-blue-600 text-white font-bold" onClick={addTag}>+</button>
-         </div>
+         {showAddTag && keyword && (
+            <AddTags
+               existingTags={allTags}
+               keywords={[keyword]}
+               closeModal={() => setShowAddTag(false)}
+               />
+         )}
       </Modal>
-
    );
 };
 

components/keywords/KeywordsTable.tsx

@@ -1,8 +1,7 @@
-import React, { useState, useMemo } from 'react';
+import React, { useState, useMemo, useCallback, useRef, useEffect } from 'react';
 import { Toaster } from 'react-hot-toast';
-import { CSSTransition } from 'react-transition-group';
-import AddKeywords from './AddKeywords';
-import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/sortFilter';
+import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
+import { filterKeywords, keywordsByDevice, sortKeywords } from '../../utils/client/sortFilter';
 import Icon from '../common/Icon';
 import Keyword from './Keyword';
 import KeywordDetails from './KeywordDetails';
@@ -10,36 +9,77 @@ import KeywordFilters from './KeywordFilter';
 import Modal from '../common/Modal';
 import { useDeleteKeywords, useFavKeywords, useRefreshKeywords } from '../../services/keywords';
 import KeywordTagManager from './KeywordTagManager';
+import AddTags from './AddTags';
+import useWindowResize from '../../hooks/useWindowResize';
+import useIsMobile from '../../hooks/useIsMobile';
+import { useUpdateSettings } from '../../services/settings';
+import { defaultSettings } from '../settings/Settings';
 
 type KeywordsTableProps = {
-   domain: Domain | null,
+   domain: DomainType | null,
    keywords: KeywordType[],
    isLoading: boolean,
    showAddModal: boolean,
-   setShowAddModal: Function
+   setShowAddModal: Function,
+   isConsoleIntegrated: boolean,
+   settings?: SettingsType
 }
 
-const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal = false, setShowAddModal }: KeywordsTableProps) => {
+const KeywordsTable = (props: KeywordsTableProps) => {
+   const titleColumnRef = useRef(null);
+   const { keywords = [], isLoading = true, isConsoleIntegrated = false, settings } = props;
+   const showSCData = isConsoleIntegrated;
    const [device, setDevice] = useState<string>('desktop');
    const [selectedKeywords, setSelectedKeywords] = useState<number[]>([]);
    const [showKeyDetails, setShowKeyDetails] = useState<KeywordType|null>(null);
    const [showRemoveModal, setShowRemoveModal] = useState<boolean>(false);
    const [showTagManager, setShowTagManager] = useState<null|number>(null);
+   const [showAddTags, setShowAddTags] = useState<boolean>(false);
+   const [SCListHeight, setSCListHeight] = useState(500);
    const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
    const [sortBy, setSortBy] = useState<string>('date_asc');
+   const [scDataType, setScDataType] = useState<string>('threeDays');
+   const [showScDataTypes, setShowScDataTypes] = useState<boolean>(false);
+   const [maxTitleColumnWidth, setMaxTitleColumnWidth] = useState(235);
    const { mutate: deleteMutate } = useDeleteKeywords(() => {});
    const { mutate: favoriteMutate } = useFavKeywords(() => {});
    const { mutate: refreshMutate } = useRefreshKeywords(() => {});
+   const [isMobile] = useIsMobile();
+
+   useWindowResize(() => {
+      setSCListHeight(window.innerHeight - (isMobile ? 200 : 400));
+      if (titleColumnRef.current) {
+         setMaxTitleColumnWidth((titleColumnRef.current as HTMLElement).clientWidth);
+      }
+   });
+
+   useEffect(() => {
+      if (titleColumnRef.current) {
+         setMaxTitleColumnWidth((titleColumnRef.current as HTMLElement).clientWidth);
+      }
+   }, [titleColumnRef]);
+
+   const tableColumns = settings?.keywordsColumns || ['Best', 'History', 'Volume', 'Search Console'];
+   const { mutate: updateMutate, isLoading: isUpdatingSettings } = useUpdateSettings(() => console.log(''));
+
+   const scDataObject:{ [k:string] : string} = {
+      threeDays: 'Last Three Days',
+      sevenDays: 'Last Seven Days',
+      thirtyDays: 'Last Thirty Days',
+      avgThreeDays: 'Last Three Days Avg',
+      avgSevenDays: 'Last Seven Days Avg',
+      avgThirtyDays: 'Last Thirty Days Avg',
+   };
 
    const processedKeywords: {[key:string] : KeywordType[]} = useMemo(() => {
       const procKeywords = keywords.filter((x) => x.device === device);
       const filteredKeywords = filterKeywords(procKeywords, filterParams);
-      const sortedKeywords = sortKeywords(filteredKeywords, sortBy);
+      const sortedKeywords = sortKeywords(filteredKeywords, sortBy, scDataType);
       return keywordsByDevice(sortedKeywords, device);
-   }, [keywords, device, sortBy, filterParams]);
+   }, [keywords, device, sortBy, filterParams, scDataType]);
 
    const allDomainTags: string[] = useMemo(() => {
-      const allTags = keywords.reduce((acc: string[], keyword) => [...acc, ...keyword.tags], []);
+      const allTags = keywords.reduce((acc: string[], keyword) => [...acc, ...keyword.tags], []).filter((t) => t && t.trim() !== '');
       return [...new Set(allTags)];
    }, [keywords]);
 
@@ -52,11 +92,44 @@ const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal =
       setSelectedKeywords(updatedSelectd);
    };
 
+   const updateColumns = (column:string) => {
+      const newColumns = tableColumns.includes(column) ? tableColumns.filter((col) => col !== column) : [...tableColumns, column];
+      updateMutate({ ...defaultSettings, ...settings, keywordsColumns: newColumns });
+   };
+
+   const shouldHideColumn = useCallback((col:string) => {
+      return settings?.keywordsColumns && !settings?.keywordsColumns.includes(col) ? 'lg:hidden' : '';
+   }, [settings?.keywordsColumns]);
+
+   const Row = ({ data, index, style }:ListChildComponentProps) => {
+      const keyword = data[index];
+      return (
+         <Keyword
+         key={keyword.ID}
+         style={style}
+         index={index}
+         selected={selectedKeywords.includes(keyword.ID)}
+         selectKeyword={selectKeyword}
+         keywordData={keyword}
+         refreshkeyword={() => refreshMutate({ ids: [keyword.ID] })}
+         favoriteKeyword={favoriteMutate}
+         manageTags={() => setShowTagManager(keyword.ID)}
+         removeKeyword={() => { setSelectedKeywords([keyword.ID]); setShowRemoveModal(true); }}
+         showKeywordDetails={() => setShowKeyDetails(keyword)}
+         lastItem={index === (processedKeywords[device].length - 1)}
+         showSCData={showSCData}
+         scDataType={scDataType}
+         tableColumns={tableColumns}
+         maxTitleColumnWidth={maxTitleColumnWidth}
+         />
+      );
+   };
+
    const selectedAllItems = selectedKeywords.length === processedKeywords[device].length;
 
    return (
       <div>
-         <div className='domKeywords flex flex-col bg-[white] rounded-md text-sm border'>
+         <div className='domKeywords flex flex-col bg-[white] rounded-md text-sm border mb-5'>
             {selectedKeywords.length > 0 && (
                <div className='font-semibold text-sm py-4 px-8 text-gray-500 '>
                   <ul className=''>
@@ -65,7 +138,7 @@ const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal =
                         className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
                         onClick={() => { refreshMutate({ ids: selectedKeywords }); setSelectedKeywords([]); }}
                         >
-                           <span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keyword
+                           <span className=' bg-indigo-100 text-blue-700 px-1 rounded'><Icon type="reload" size={11} /></span> Refresh Keywords
                         </a>
                      </li>
                      <li className='inline-block mr-4'>
@@ -73,7 +146,14 @@ const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal =
                         className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
                         onClick={() => setShowRemoveModal(true)}
                         >
-                           <span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keyword</a>
+                           <span className=' bg-red-100 text-red-600 px-1 rounded'><Icon type="trash" size={14} /></span> Remove Keywords</a>
+                     </li>
+                     <li className='inline-block mr-4'>
+                        <a
+                        className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
+                        onClick={() => setShowAddTags(true)}
+                        >
+                           <span className=' bg-green-100 text-green-500  px-1 rounded'><Icon type="tags" size={14} /></span> Tag Keywords</a>
                      </li>
                   </ul>
                </div>
@@ -88,13 +168,19 @@ const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal =
                   keywords={keywords}
                   device={device}
                   setDevice={setDevice}
+                  updateColumns={updateColumns}
+                  tableColumns={tableColumns}
+                  integratedConsole={isConsoleIntegrated}
                />
             )}
-            <div className='styled-scrollbar w-full overflow-auto min-h-[60vh] '>
+            <div className={`domkeywordsTable domkeywordsTable--keywords 
+            ${showSCData && tableColumns.includes('Search Console') ? 'domkeywordsTable--hasSC' : ''} 
+               styled-scrollbar w-full overflow-auto min-h-[60vh]`}>
                <div className=' lg:min-w-[800px]'>
                   <div className={`domKeywords_head domKeywords_head--${sortBy} hidden lg:flex p-3 px-6 bg-[#FCFCFF]
                    text-gray-600 justify-between items-center font-semibold border-y`}>
-                     <span className='domKeywords_head_keyword flex-1 basis-20 w-auto '>
+                     <span ref={titleColumnRef} className={`domKeywords_head_keyword flex-1 basis-[4rem] w-auto lg:flex-1 
+                        ${showSCData && tableColumns.includes('Search Console') ? 'lg:basis-20' : 'lg:basis-10'} lg:w-auto lg:flex lg:items-center `}>
                      {processedKeywords[device].length > 0 && (
                         <button
                            className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px]  border border-slate-300 
@@ -104,32 +190,71 @@ const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal =
                               <Icon type="check" size={10} />
                         </button>
                      )}
-                        Keyword
+                  {/* ${showSCData ? 'lg:min-w-[220px]' : 'lg:min-w-[280px]'} */}
+                        <span className={`inline-block lg:flex lg:items-center 
+                           ${showSCData && tableColumns.includes('Search Console') ? 'lg:max-w-[235px]' : ''}`}>
+                           Keyword
+                        </span>
                      </span>
-                     <span className='domKeywords_head_position flex-1 basis-40 grow-0 text-center'>Position</span>
-                     <span className='domKeywords_head_history flex-1'>History (7d)</span>
+                     <span className='domKeywords_head_position flex-1 basis-24 grow-0 text-center'>Position</span>
+                     <span className={`domKeywords_head_best flex-1 basis-16 grow-0 text-center  ${shouldHideColumn('Best')}`}>Best</span>
+                     <span className={`domKeywords_head_history flex-1 basis-20 grow-0  ${shouldHideColumn('History')}`}>History (7d)</span>
+                     <span className={`domKeywords_head_volume flex-1 basis-24 grow-0 text-center ${shouldHideColumn('Volume')}`}>Volume</span>
                      <span className='domKeywords_head_url flex-1'>URL</span>
-                     <span className='domKeywords_head_updated flex-1'>Updated</span>
+                     <span className='domKeywords_head_updated flex-1 relative left-3 max-w-[150px]'>Updated</span>
+                     {showSCData && tableColumns.includes('Search Console') && (
+                        <div className='domKeywords_head_sc flex-1 min-w-[170px] lg:max-w-[170px] mr-7 text-center'>
+                           {/* Search Console */}
+                           <div>
+                              <div
+                              className=' w-48 select-none cursor-pointer absolute bg-white rounded-full
+                              px-2 py-[2px] mt-[-22px] ml-3 border border-gray-200 z-40'
+                              onClick={() => setShowScDataTypes(!showScDataTypes)}>
+                                 <Icon type="google" size={13} /> {scDataObject[scDataType]}
+                                 <Icon classes="ml-2" type={showScDataTypes ? 'caret-up' : 'caret-down'} size={10} />
+                              </div>
+                              {showScDataTypes && (
+                                 <div className='absolute bg-white border border-gray-200 z-50 w-44 rounded mt-2 ml-5 text-gray-500'>
+                                    {Object.keys(scDataObject).map((itemKey) => {
+                                       return <span
+                                                className={`block p-2 cursor-pointer hover:bg-indigo-50 hover:text-indigo-600
+                                                 ${scDataType === itemKey ? 'bg-indigo-100 text-indigo-600' : ''}`}
+                                                key={itemKey}
+                                                onClick={() => { setScDataType(itemKey); setShowScDataTypes(false); }}>
+                                                   {scDataObject[itemKey]}
+                                                </span>;
+                                    })}
+                                 </div>
+                              )}
+                           </div>
+                           <div className='relative top-2 flex justify-between'>
+                              <span className='min-w-[40px]'>Pos</span>
+                              <span className='min-w-[40px]'>Imp</span>
+                              <span className='min-w-[40px]'>Visits</span>
+                              {/* <span>CTR</span> */}
+                           </div>
+                        </div>
+                     )}
                   </div>
-                  <div className='domKeywords_keywords border-gray-200'>
-                     {processedKeywords[device] && processedKeywords[device].length > 0
-                        && processedKeywords[device].map((keyword, index) => <Keyword
-                                                      key={keyword.ID}
-                                                      selected={selectedKeywords.includes(keyword.ID)}
-                                                      selectKeyword={selectKeyword}
-                                                      keywordData={keyword}
-                                                      refreshkeyword={() => refreshMutate({ ids: [keyword.ID] })}
-                                                      favoriteKeyword={favoriteMutate}
-                                                      manageTags={() => setShowTagManager(keyword.ID)}
-                                                      removeKeyword={() => { setSelectedKeywords([keyword.ID]); setShowRemoveModal(true); }}
-                                                      showKeywordDetails={() => setShowKeyDetails(keyword)}
-                                                      lastItem={index === (processedKeywords[device].length - 1)}
-                                                      />)}
+                  <div className='domKeywords_keywords border-gray-200 min-h-[55vh] relative'>
+                     {processedKeywords[device] && processedKeywords[device].length > 0 && (
+                        <List
+                        innerElementType="div"
+                        itemData={processedKeywords[device]}
+                        itemCount={processedKeywords[device].length}
+                        itemSize={isMobile ? 146 : 57}
+                        height={SCListHeight}
+                        width={'100%'}
+                        className={'styled-scrollbar'}
+                        >
+                           {Row}
+                        </List>
+                     )}
                      {!isLoading && processedKeywords[device].length === 0 && (
-                        <p className=' p-9 mt-[10%] text-center text-gray-500'>No Keywords Added for this Device Type.</p>
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>No Keywords Added for this Device Type.</p>
                      )}
                      {isLoading && (
-                        <p className=' p-9 mt-[10%] text-center text-gray-500'>Loading Keywords...</p>
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>Loading Keywords...</p>
                      )}
                   </div>
                </div>
@@ -157,13 +282,6 @@ const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal =
                   </div>
             </Modal>
          )}
-         <CSSTransition in={showAddModal} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddKeywords
-               domain={domain?.domain || ''}
-               keywords={keywords}
-               closeModal={() => setShowAddModal(false)}
-               />
-         </CSSTransition>
          {showTagManager && (
             <KeywordTagManager
                allTags={allDomainTags}
@@ -171,6 +289,13 @@ const KeywordsTable = ({ domain, keywords = [], isLoading = true, showAddModal =
                closeModal={() => setShowTagManager(null)}
                />
          )}
+         {showAddTags && (
+            <AddTags
+               existingTags={allDomainTags}
+               keywords={keywords.filter((k) => selectedKeywords.includes(k.ID))}
+               closeModal={() => setShowAddTags(false)}
+               />
+         )}
          <Toaster position='bottom-center' containerClassName="react_toaster" />
       </div>
    );

components/keywords/SCKeyword.tsx

@@ -0,0 +1,72 @@
+import React from 'react';
+import Icon from '../common/Icon';
+import countries from '../../utils/countries';
+import KeywordPosition from './KeywordPosition';
+import { formattedNum } from '../../utils/client/helpers';
+
+type SCKeywordProps = {
+   keywordData: SearchAnalyticsItem,
+   selected: boolean,
+   selectKeyword: Function,
+   lastItem?:boolean,
+   isTracked: boolean,
+   style: Object
+}
+
+const SCKeyword = (props: SCKeywordProps) => {
+   const { keywordData, selected, lastItem, selectKeyword, style, isTracked = false } = props;
+   const { keyword, uid, position, country, impressions, ctr, clicks } = keywordData;
+
+   return (
+      <div
+      key={keyword}
+      style={style}
+      className={`keyword relative py-5 px-4 text-gray-600 border-b-[1px] border-gray-200 lg:py-4 lg:px-6 lg:border-0 
+      lg:flex lg:justify-between lg:items-center ${selected ? ' bg-indigo-50 keyword--selected' : ''} ${lastItem ? 'border-b-0' : ''}`}>
+
+         <div className=' w-3/4 lg:flex-1 lg:basis-20 lg:w-auto font-semibold cursor-pointer'>
+            <button
+               className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px] border 
+               ${isTracked || selected ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}
+               ${isTracked ? 'bg-gray-400 border-gray-400 cursor-default' : ''}`}
+               onClick={() => !isTracked && selectKeyword(uid)}
+               >
+                  <Icon type="check" size={10} title={isTracked ? 'Already in Tracker' : ''} />
+            </button>
+            <a className='py-2 hover:text-blue-600'>
+               <span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country] && countries[country][0]} />{keyword}
+            </a>
+         </div>
+
+         <div className={`keyword_position absolute bg-[#f8f9ff] w-fit min-w-[50px] h-15 p-2 text-base mt-[-20px] rounded right-5 lg:relative
+          lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-40 lg:grow-0 lg:right-0 text-center font-semibold`}>
+            <KeywordPosition position={position} />
+            <span className='block text-xs text-gray-500 lg:hidden'>Position</span>
+         </div>
+
+         <div className='keyword_imp text-center inline-block lg:flex-1 '>
+            <span className='mr-3 lg:hidden'>
+               <Icon type="eye" size={14} color="#999" />
+            </span>
+            {formattedNum(impressions)}
+         </div>
+
+         <div className={'keyword_visits text-center inline-block mt-4 mr-5 ml-5 lg:flex-1 lg:m-0 max-w-[70px] lg:max-w-none lg:pr-5'}>
+            <span className='mr-3 lg:hidden'>
+               <Icon type="cursor" size={14} color="#999" />
+            </span>
+            {formattedNum(clicks)}
+         </div>
+
+         <div className='keyword_ctr text-center inline-block mt-4 relative lg:flex-1 lg:m-0 '>
+            <span className='mr-3 lg:hidden'>
+               <Icon type="target" size={14} color="#999" />
+            </span>
+            {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(ctr)}%
+         </div>
+
+      </div>
+   );
+ };
+
+ export default SCKeyword;

components/keywords/SCKeywordsTable.tsx

@@ -0,0 +1,224 @@
+import { useRouter } from 'next/router';
+import React, { useState, useMemo } from 'react';
+import { Toaster } from 'react-hot-toast';
+import { FixedSizeList as List, ListChildComponentProps } from 'react-window';
+import { useAddKeywords, useFetchKeywords } from '../../services/keywords';
+import { SCfilterKeywords, SCkeywordsByDevice, SCsortKeywords } from '../../utils/client/SCsortFilter';
+import Icon from '../common/Icon';
+import KeywordFilters from './KeywordFilter';
+import SCKeyword from './SCKeyword';
+import useWindowResize from '../../hooks/useWindowResize';
+import useIsMobile from '../../hooks/useIsMobile';
+import { formattedNum } from '../../utils/client/helpers';
+
+type SCKeywordsTableProps = {
+   domain: DomainType | null,
+   keywords: SearchAnalyticsItem[],
+   isLoading: boolean,
+   isConsoleIntegrated: boolean,
+}
+
+type SCCountryDataType = {
+   keywords: number,
+   impressions: number,
+   visits: number
+}
+
+const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleIntegrated = true }: SCKeywordsTableProps) => {
+   const router = useRouter();
+   const [device, setDevice] = useState<string>('desktop');
+   const [selectedKeywords, setSelectedKeywords] = useState<string[]>([]);
+   const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
+   const [sortBy, setSortBy] = useState<string>('imp_desc');
+   const [SCListHeight, setSCListHeight] = useState(500);
+   const { keywordsData } = useFetchKeywords(router, domain?.domain || '');
+   const addedkeywords: string[] = keywordsData?.keywords?.map((key: KeywordType) => `${key.keyword}:${key.country}:${key.device}`) || [];
+   const { mutate: addKeywords } = useAddKeywords(() => { if (domain && domain.slug) router.push(`/domain/${domain.slug}`); });
+   const [isMobile] = useIsMobile();
+   useWindowResize(() => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400)));
+
+   const finalKeywords: {[key:string] : SCKeywordType[] } = useMemo(() => {
+      const procKeywords = keywords.filter((x) => x.device === device);
+      const filteredKeywords = SCfilterKeywords(procKeywords, filterParams);
+      const sortedKeywords = SCsortKeywords(filteredKeywords, sortBy);
+      return SCkeywordsByDevice(sortedKeywords, device);
+   }, [keywords, device, filterParams, sortBy]);
+
+   const SCCountryData: {[key:string] : SCCountryDataType } = useMemo(() => {
+      const countryData:{[key:string] : SCCountryDataType } = {};
+
+      Object.keys(finalKeywords).forEach((dateKey) => {
+         finalKeywords[dateKey].forEach((keyword) => {
+            const kCountry = keyword.country;
+            if (!countryData[kCountry]) { countryData[kCountry] = { keywords: 0, impressions: 0, visits: 0 }; }
+            countryData[kCountry].keywords += 1;
+            countryData[kCountry].visits += (keyword.clicks || 0);
+            countryData[kCountry].impressions += (keyword.impressions || 0);
+         });
+      });
+
+      return countryData;
+   }, [finalKeywords]);
+
+   const viewSummary: {[key:string] : number } = useMemo(() => {
+         const keyCount = finalKeywords[device].length;
+         const kwSummary = { position: 0, impressions: 0, visits: 0, ctr: 0 };
+         finalKeywords[device].forEach((k) => {
+            kwSummary.position += k.position;
+            kwSummary.impressions += k.impressions;
+            kwSummary.visits += k.clicks;
+            kwSummary.ctr += k.ctr;
+         });
+         return {
+            ...kwSummary,
+            position: Math.round(kwSummary.position / keyCount),
+            ctr: kwSummary.ctr / keyCount,
+         };
+   }, [finalKeywords, device]);
+
+   const selectKeyword = (keywordID: string) => {
+      console.log('Select Keyword: ', keywordID);
+      let updatedSelectd = [...selectedKeywords, keywordID];
+      if (selectedKeywords.includes(keywordID)) {
+         updatedSelectd = selectedKeywords.filter((keyID) => keyID !== keywordID);
+      }
+      setSelectedKeywords(updatedSelectd);
+   };
+
+   const addSCKeywordsToTracker = () => {
+      const selectedkeywords:KeywordAddPayload[] = [];
+      keywords.forEach((kitem:SCKeywordType) => {
+         if (selectedKeywords.includes(kitem.uid)) {
+            const { keyword, country } = kitem;
+            selectedkeywords.push({ keyword, device, country, domain: domain?.domain || '', tags: '' });
+         }
+      });
+      addKeywords(selectedkeywords);
+      setSelectedKeywords([]);
+   };
+
+   const selectedAllItems = selectedKeywords.length === finalKeywords[device].length;
+
+   const Row = ({ data, index, style }:ListChildComponentProps) => {
+      const keyword = data[index];
+      return (
+         <SCKeyword
+         key={keyword.uid}
+         style={style}
+         selected={selectedKeywords.includes(keyword.uid)}
+         selectKeyword={selectKeyword}
+         keywordData={keyword}
+         isTracked={addedkeywords.includes(`${keyword.keyword}:${keyword.country}:${keyword.device}`)}
+         lastItem={index === (finalKeywords[device].length - 1)}
+         />
+   );
+};
+
+   return (
+      <div>
+         <div className='domKeywords flex flex-col bg-[white] rounded-md text-sm border mb-4'>
+            {selectedKeywords.length > 0 && (
+               <div className='font-semibold text-sm py-4 px-8 text-gray-500 '>
+                  <ul className=''>
+                     <li className='inline-block mr-4'>
+                        <a
+                        className='block px-2 py-2 cursor-pointer hover:text-indigo-600'
+                        onClick={() => addSCKeywordsToTracker()}
+                        >
+                           <span className=' bg-indigo-100 text-blue-700 px-1 rounded font-black'>+</span> Add Keywords to Tracker
+                        </a>
+                     </li>
+                  </ul>
+               </div>
+            )}
+            {selectedKeywords.length === 0 && (
+               <KeywordFilters
+                  allTags={[]}
+                  filterParams={filterParams}
+                  filterKeywords={(params:KeywordFilters) => setFilterParams(params)}
+                  updateSort={(sorted:string) => setSortBy(sorted)}
+                  sortBy={sortBy}
+                  keywords={keywords}
+                  device={device}
+                  setDevice={setDevice}
+                  isConsole={true}
+                  integratedConsole={isConsoleIntegrated}
+                  SCcountries={Object.keys(SCCountryData)}
+               />
+            )}
+            <div className='domkeywordsTable domkeywordsTable--sckeywords styled-scrollbar w-full overflow-auto min-h-[60vh]'>
+               <div className=' lg:min-w-[800px]'>
+                  <div className={`domKeywords_head domKeywords_head--${sortBy} hidden lg:flex p-3 px-6 bg-[#FCFCFF]
+                   text-gray-600 justify-between items-center font-semibold border-y`}>
+                     <span className='domKeywords_head_keyword flex-1 basis-20 w-auto '>
+                     {finalKeywords[device].length > 0 && (
+                        <button
+                           className={`p-0 mr-2 leading-[0px] inline-block rounded-sm pt-0 px-[1px] pb-[3px]  border border-slate-300 
+                           ${selectedAllItems ? ' bg-blue-700 border-blue-700 text-white' : 'text-transparent'}`}
+                           onClick={() => setSelectedKeywords(selectedAllItems ? [] : finalKeywords[device].map((k: SearchAnalyticsItem) => k.uid))}
+                           >
+                              <Icon type="check" size={10} />
+                        </button>
+                     )}
+                        Keyword
+                     </span>
+                     <span className='domKeywords_head_position flex-1 basis-40 grow-0 text-center'>Position</span>
+                     <span className='domKeywords_head_imp flex-1 text-center'>Impressions</span>
+                     <span className='domKeywords_head_visits flex-1 text-center'>Visits</span>
+                     <span className='domKeywords_head_ctr flex-1 text-center'>CTR</span>
+                  </div>
+                  <div className='domKeywords_keywords border-gray-200 min-h-[55vh] relative' data-domain={domain?.domain}>
+                     {!isLoading && finalKeywords[device] && finalKeywords[device].length > 0 && (
+                        <List
+                        innerElementType="div"
+                        itemData={finalKeywords[device]}
+                        itemCount={finalKeywords[device].length}
+                        itemSize={isMobile ? 100 : 57}
+                        height={SCListHeight}
+                        width={'100%'}
+                        className={'styled-scrollbar'}
+                        >
+                           {Row}
+                        </List>
+                     )}
+                     {!isLoading && finalKeywords[device] && finalKeywords[device].length > 0 && (
+                        <div className={`domKeywords_head hidden lg:flex p-3 px-6 bg-[#FCFCFF]
+                           text-gray-600 justify-between items-center font-semibold border-y`}>
+                              <span className='domKeywords_head_keyword flex-1 basis-20 w-auto font-semibold'>
+                                 {finalKeywords[device].length} {device} Keywords
+                              </span>
+                              <span className='domKeywords_head_position flex-1 basis-40 grow-0 text-center'>{viewSummary.position}</span>
+                              <span className='domKeywords_head_imp flex-1 text-center'>
+                                 {formattedNum(viewSummary.impressions)}
+                              </span>
+                              <span className='domKeywords_head_visits flex-1 text-center'>
+                                 {formattedNum(viewSummary.visits)}
+                              </span>
+                              <span className='domKeywords_head_ctr flex-1 text-center'>
+                                 {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(viewSummary.ctr)}%
+                              </span>
+                        </div>
+                     )}
+                     {isConsoleIntegrated && !isLoading && finalKeywords[device].length === 0 && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>
+                           Could Not fetch Keyword Data for this Domain from Google Search Console.
+                        </p>
+                     )}
+                     {isConsoleIntegrated && isLoading && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>Loading Keywords...</p>
+                     )}
+                     {!isConsoleIntegrated && (
+                        <p className=' p-9 pt-[10%] text-center text-gray-500'>
+                        Google Search Console has not been Integrated yet. Please follow <a className='text-indigo-600 underline' href='https://docs.serpbear.com/miscellaneous/integrate-google-search-console' target="_blank" rel='noreferrer'>These Steps</a> to integrate Google Search Data for this Domain.
+                        </p>
+                     )}
+                  </div>
+               </div>
+            </div>
+         </div>
+         <Toaster position='bottom-center' containerClassName="react_toaster" />
+      </div>
+   );
+ };
+
+ export default SCKeywordsTable;

components/settings/AdWordsSettings.tsx

@@ -0,0 +1,146 @@
+import React from 'react';
+import { useMutateKeywordsVolume, useTestAdwordsIntegration } from '../../services/adwords';
+import Icon from '../common/Icon';
+import SecretField from '../common/SecretField';
+
+type AdWordsSettingsProps = {
+   settings: SettingsType,
+   settingsError: null | {
+      type: string,
+      msg: string
+   },
+   updateSettings: Function,
+   performUpdate: Function,
+   closeSettings: Function
+}
+
+const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdate, closeSettings }:AdWordsSettingsProps) => {
+   const {
+      adwords_client_id = '',
+      adwords_client_secret = '',
+      adwords_developer_token = '',
+      adwords_account_id = '',
+      adwords_refresh_token = '',
+   } = settings || {};
+
+   const { mutate: testAdWordsIntegration, isLoading: isTesting } = useTestAdwordsIntegration();
+   const { mutate: getAllVolumeData, isLoading: isUpdatingVolume } = useMutateKeywordsVolume();
+
+   const cloudProjectIntegrated = adwords_client_id && adwords_client_secret && adwords_refresh_token;
+   const hasAllCredentials = adwords_client_id && adwords_client_secret && adwords_refresh_token && adwords_developer_token && adwords_account_id;
+
+   const udpateAndAuthenticate = () => {
+      if (adwords_client_id && adwords_client_secret) {
+         const link = document.createElement('a');
+         link.href = `https://accounts.google.com/o/oauth2/v2/auth/oauthchooseaccount?access_type=offline&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fadwords&response_type=code&client_id=${adwords_client_id}&redirect_uri=${`${encodeURIComponent(window.location.origin)}/api/adwords`}&service=lso&o2v=2&theme=glif&flowName=GeneralOAuthFlow`;
+         link.target = '_blank';
+         link.click();
+         if (performUpdate) {
+            performUpdate();
+            closeSettings();
+         }
+      }
+   };
+
+   const testIntegration = () => {
+      if (hasAllCredentials) {
+         testAdWordsIntegration({ developer_token: adwords_developer_token, account_id: adwords_account_id });
+      }
+   };
+
+   const updateVolumeData = () => {
+      if (hasAllCredentials) {
+         getAllVolumeData({ domain: 'all' });
+      }
+   };
+
+   return (
+      <div>
+      <div>
+         <div className=' border-t border-gray-100 pt-4 pb-0'>
+            <h4 className=' mb-3 font-semibold text-blue-700'>Step 1: Connect Google Cloud Project</h4>
+            <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+               <SecretField
+               label='Client ID'
+               onChange={(client_id:string) => updateSettings('adwords_client_id', client_id)}
+               value={adwords_client_id}
+               placeholder='3943006-231f65cjm.apps.googleusercontent.com'
+               />
+            </div>
+            <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+               <SecretField
+               label='Client Secret'
+               onChange={(client_secret:string) => updateSettings('adwords_client_secret', client_secret)}
+               value={adwords_client_secret}
+               placeholder='GTXSPX-45asaf-u1s252sd6qdE9yc8T'
+               />
+            </div>
+            <button
+            className={`py-2 px-5 w-full text-sm font-semibold rounded  bg-indigo-50 text-blue-700 border border-indigo-100
+            ${adwords_client_id && adwords_client_secret ? 'cursor-pointer' : ' cursor-not-allowed opacity-40'}
+             hover:bg-blue-700 hover:text-white transition`}
+            title='Insert All the data in the above fields to Authenticate'
+            onClick={udpateAndAuthenticate}>
+               <Icon type='google' size={14} /> {adwords_refresh_token ? 'Re-Authenticate' : 'Authenticate'} Integration
+            </button>
+         </div>
+         <div className='mt-4 border-t mb-4 border-b border-gray-100 pt-4 pb-0 relative'>
+            {!cloudProjectIntegrated && <div className=' absolute w-full h-full z-50' />}
+            <h4 className=' mb-3 font-semibold text-blue-700'>Step 2: Connect Google Ads</h4>
+            <div className={!cloudProjectIntegrated ? 'opacity-40' : ''}>
+               <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+                  <SecretField
+                  label='Developer Token'
+                  onChange={(developer_token:string) => updateSettings('adwords_developer_token', developer_token)}
+                  value={adwords_developer_token}
+                  placeholder='4xr6jY94kAxtXk4rfcgc4w'
+                  />
+               </div>
+               <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+                  <SecretField
+                  label='Test Account ID'
+                  onChange={(account_id:string) => updateSettings('adwords_account_id', account_id)}
+                  value={adwords_account_id}
+                  placeholder='590-948-9101'
+                  />
+               </div>
+               <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+                  <button
+                  className={`py-2 px-5 w-full text-sm font-semibold rounded bg-indigo-50 text-blue-700 border border-indigo-100
+                  ${hasAllCredentials ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
+                  hover:bg-blue-700 hover:text-white transition`}
+                  title={hasAllCredentials ? '' : 'Insert All the data in the above fields to Test the Integration'}
+                  onClick={testIntegration}>
+                     {isTesting && <Icon type='loading' />}
+                     <Icon type='adwords' size={14} /> Test Google Ads Integration
+                  </button>
+               </div>
+            </div>
+         </div>
+         <div className='mt-4 mb-4 border-b border-gray-100 pt-4 pb-0 relative'>
+            {!hasAllCredentials && <div className=' absolute w-full h-full z-50' />}
+            <h4 className=' mb-3 font-semibold text-blue-700'>Update Keyword Volume Data</h4>
+            <div className={!hasAllCredentials ? 'opacity-40' : ''}>
+               <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+                  <p>Update Volume data for all your Tracked Keywords.</p>
+               </div>
+               <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+                  <button
+                  className={`py-2 px-5 w-full text-sm font-semibold rounded bg-indigo-50 text-blue-700 border border-indigo-100
+                  ${hasAllCredentials ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
+                  hover:bg-blue-700 hover:text-white transition`}
+                  onClick={updateVolumeData}>
+                     <Icon type={isUpdatingVolume ? 'loading' : 'reload'} size={isUpdatingVolume ? 16 : 12} /> Update Keywords Volume Data
+                  </button>
+               </div>
+            </div>
+         </div>
+         <p className='mb-4 text-xs'>
+            Relevant Documentation: <a target='_blank' rel='noreferrer' href='https://docs.serpbear.com/miscellaneous/integrate-google-ads' className=' underline text-blue-600'>Integrate Google Ads</a>.
+         </p>
+      </div>
+   </div>
+   );
+};
+
+export default AdWordsSettings;

components/settings/Changelog.tsx

@@ -0,0 +1,80 @@
+import React, { useLayoutEffect, useMemo } from 'react';
+import TimeAgo from 'react-timeago';
+import dayjs from 'dayjs';
+import SidePanel from '../common/SidePanel';
+import { useFetchChangelog } from '../../services/misc';
+import Icon from '../common/Icon';
+
+const Markdown = React.lazy(() => import('react-markdown'));
+
+type ChangeLogProps = {
+   closeChangeLog: Function,
+}
+
+const ChangeLogloader = () => {
+   return (
+      <div className='w-full h-full absolute flex justify-center items-center'>
+         <Icon type="loading" size={36} color='#999' />
+      </div>
+   );
+};
+
+const ChangeLog = ({ closeChangeLog }: ChangeLogProps) => {
+   const { data: changeLogData, isLoading } = useFetchChangelog();
+
+   useLayoutEffect(() => {
+      document.body.style.overflow = 'hidden';
+      // eslint-disable-next-line no-unused-expressions
+      () => {
+         console.log('run CleanUp !');
+         document.body.style.overflow = 'auto';
+       };
+   }, []);
+
+   const onClose = () => {
+      document.body.style.overflow = 'auto';
+      closeChangeLog();
+   };
+
+   const changeLogs = useMemo(() => {
+      if (changeLogData && Array.isArray(changeLogData)) {
+         return changeLogData.map(({ name = '', body, published_at }:{name: string, body: string, published_at: string}) => ({
+            version: name,
+            major: !!(name.match(/v\d+\.0+\.0/)),
+            date: published_at,
+            content: body.replaceAll(/^(##|###) \[([^\]]+)\]\(([^)]+)\) \(([^)]+)\)/g, '')
+            .replaceAll(/\(\[(.*?)\]\((https:\/\/github\.com\/towfiqi\/serpbear\/commit\/([a-f0-9]{40}))\)\)/g, ''),
+         }));
+      }
+      return [];
+   }, [changeLogData]);
+
+   return <SidePanel title='SerpBear Changelog' closePanel={onClose}>
+            <React.Suspense fallback={<ChangeLogloader />}>
+               {!isLoading && changeLogs.length > 0 && (
+                  <div className='changelog-body bg-[#f8f9ff] px-6 pt-4 pb-10 overflow-y-auto styled-scrollbar'>
+                     {changeLogs.map(({ version, content, date, major }) => {
+                        return (
+                           <div
+                           key={version}
+                           className={`domKeywords bg-white rounded mb-6 border ${major ? ' border-indigo-400' : 'border-transparent'}`}>
+                              <h4 className=' px-5 py-3 border-b border-b-gray-100 flex justify-between text-indigo-700 font-semibold'>
+                                 <a href={`https://github.com/towfiqi/serpbear/releases/tag/${version}`}>
+                                    {version} {major && <span className=' text-xs bg-amber-100 text-amber-700 px-2 py-1 rounded ml-2'>Major</span>}
+                                 </a>
+                                 <span className=' text-sm text-gray-500'>
+                                    <TimeAgo title={dayjs(date).format('DD-MMM-YYYY, hh:mm:ss A')} date={date} />
+                                 </span>
+                              </h4>
+                              <div className='changelog-content px-5 py-3 text-sm text-left'><Markdown>{content}</Markdown></div>
+                           </div>
+                        );
+                     })}
+                  </div>
+               )}
+               {isLoading && <ChangeLogloader />}
+            </React.Suspense>
+         </SidePanel>;
+};
+
+export default ChangeLog;

components/settings/IntegrationSettings.tsx

@@ -0,0 +1,53 @@
+import React, { useState } from 'react';
+import SearchConsoleSettings from './SearchConsoleSettings';
+import AdWordsSettings from './AdWordsSettings';
+import Icon from '../common/Icon';
+
+type IntegrationSettingsProps = {
+   settings: SettingsType,
+   settingsError: null | {
+      type: string,
+      msg: string
+   },
+   updateSettings: Function,
+   performUpdate: Function,
+   closeSettings: Function
+}
+const IntegrationSettings = ({ settings, settingsError, updateSettings, performUpdate, closeSettings }:IntegrationSettingsProps) => {
+   const [currentTab, setCurrentTab] = useState<string>('searchconsole');
+   const tabStyle = 'inline-block px-4 py-1 rounded-full mr-3 cursor-pointer text-sm';
+   return (
+      <div className='settings__content styled-scrollbar p-6 text-sm'>
+         <div className='mb-4 '>
+            <ul>
+               <li
+               className={`${tabStyle} ${currentTab === 'searchconsole' ? ' bg-blue-50 text-blue-600' : ''}`}
+               onClick={() => setCurrentTab('searchconsole')}>
+                  <Icon type='google' size={14} /> Search Console
+               </li>
+               <li
+               className={`${tabStyle} ${currentTab === 'adwords' ? ' bg-blue-50 text-blue-600' : ''}`}
+               onClick={() => setCurrentTab('adwords')}>
+                  <Icon type='adwords' size={14} /> Google Ads
+               </li>
+            </ul>
+         </div>
+         <div>
+            {currentTab === 'searchconsole' && settings && (
+               <SearchConsoleSettings settings={settings} updateSettings={updateSettings} settingsError={settingsError} />
+            )}
+            {currentTab === 'adwords' && settings && (
+               <AdWordsSettings
+               settings={settings}
+               updateSettings={updateSettings}
+               settingsError={settingsError}
+               performUpdate={performUpdate}
+               closeSettings={closeSettings}
+               />
+            )}
+         </div>
+      </div>
+   );
+};
+
+export default IntegrationSettings;

components/settings/NotificationSettings.tsx

@@ -0,0 +1,114 @@
+import React from 'react';
+import SelectField from '../common/SelectField';
+import SecretField from '../common/SecretField';
+import InputField from '../common/InputField';
+
+type NotificationSettingsProps = {
+   settings: SettingsType,
+   settingsError: null | {
+      type: string,
+      msg: string
+   },
+   updateSettings: Function,
+}
+
+const NotificationSettings = ({ settings, settingsError, updateSettings }:NotificationSettingsProps) => {
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+
+   return (
+      <div>
+         <div className='settings__content styled-scrollbar p-6 text-sm'>
+            <div className="settings__section__input mb-5">
+               <SelectField
+               label='Notification Frequency'
+                  multiple={false}
+                  selected={[settings.notification_interval]}
+                  options={[
+                     { label: 'Daily', value: 'daily' },
+                     { label: 'Weekly', value: 'weekly' },
+                     { label: 'Monthly', value: 'monthly' },
+                     { label: 'Never', value: 'never' },
+                  ]}
+                  defaultLabel={'Notification Settings'}
+                  updateField={(updated:string[]) => updated[0] && updateSettings('notification_interval', updated[0])}
+                  rounded='rounded'
+                  maxHeight={48}
+                  minWidth={220}
+               />
+            </div>
+            {settings.notification_interval !== 'never' && (
+               <>
+                  <div className="settings__section__input mb-5">
+                     <InputField
+                     label='Notification Emails'
+                     hasError={settingsError?.type === 'no_email'}
+                     value={settings?.notification_email}
+                     placeholder={'test@gmail.com, test2@test.com'}
+                     onChange={(value:string) => updateSettings('notification_email', value)}
+                     />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                     <InputField
+                     label='SMTP Server'
+                     hasError={settingsError?.type === 'no_smtp_server'}
+                     value={settings?.smtp_server || ''}
+                     placeholder={'test@gmail.com, test2@test.com'}
+                     onChange={(value:string) => updateSettings('smtp_server', value)}
+                     />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                     <InputField
+                     label='SMTP Port'
+                     hasError={settingsError?.type === 'no_smtp_port'}
+                     value={settings?.smtp_port || ''}
+                     placeholder={'2234'}
+                     onChange={(value:string) => updateSettings('smtp_port', value)}
+                     />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                     <InputField
+                        label='SMTP Username'
+                        hasError={settingsError?.type === 'no_smtp_port'}
+                        value={settings?.smtp_username || ''}
+                        onChange={(value:string) => updateSettings('smtp_username', value)}
+                        />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                     <SecretField
+                     label='SMTP Password'
+                     value={settings?.smtp_password || ''}
+                     onChange={(value:string) => updateSettings('smtp_password', value)}
+                     />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                        <InputField
+                        label='From Email Address'
+                        hasError={settingsError?.type === 'no_smtp_from'}
+                        value={settings?.notification_email_from || ''}
+                        placeholder="no-reply@mydomain.com"
+                        onChange={(value:string) => updateSettings('notification_email_from', value)}
+                        />
+                  </div>
+                  <div className="settings__section__input mb-5">
+                        <InputField
+                        label='Email From Name'
+                        hasError={settingsError?.type === 'no_smtp_from'}
+                        value={settings?.notification_email_from_name || 'Serpbear'}
+                        placeholder="Serpbear"
+                        onChange={(value:string) => updateSettings('notification_email_from_name', value)}
+                        />
+                  </div>
+               </>
+            )}
+
+            </div>
+            {settingsError?.msg && (
+               <div className='absolute w-full bottom-16  text-center p-3 bg-red-100 text-red-600 text-sm font-semibold'>
+                  {settingsError.msg}
+               </div>
+            )}
+      </div>
+   );
+};
+
+export default NotificationSettings;

components/settings/ScraperSettings.tsx

@@ -0,0 +1,137 @@
+import React from 'react';
+import { useClearFailedQueue } from '../../services/settings';
+import Icon from '../common/Icon';
+import SelectField, { SelectionOption } from '../common/SelectField';
+import SecretField from '../common/SecretField';
+import ToggleField from '../common/ToggleField';
+
+type ScraperSettingsProps = {
+   settings: SettingsType,
+   settingsError: null | {
+      type: string,
+      msg: string
+   },
+   updateSettings: Function,
+}
+
+const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSettingsProps) => {
+   const { mutate: clearFailedMutate, isLoading: clearingQueue } = useClearFailedQueue(() => {});
+
+   const scrapingOptions: SelectionOption[] = [
+      { label: 'Daily', value: 'daily' },
+      { label: 'Every Other Day', value: 'other_day' },
+      { label: 'Weekly', value: 'weekly' },
+      { label: 'Monthly', value: 'monthly' },
+      { label: 'Never', value: 'never' },
+   ];
+   const delayOptions: SelectionOption[] = [
+      { label: 'No Delay', value: '0' },
+      { label: '5 Seconds', value: '5000' },
+      { label: '10 Seconds', value: '10000' },
+      { label: '30 Seconds', value: '30000' },
+      { label: '1 Minutes', value: '60000' },
+      { label: '2 Minutes', value: '120000' },
+      { label: '5 Minutes', value: '300000' },
+      { label: '10 Minutes', value: '600000' },
+      { label: '15 Minutes', value: '900000' },
+      { label: '30 Minutes', value: '1800000' },
+   ];
+   const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : [];
+   const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers];
+   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+
+   return (
+      <div>
+      <div className='settings__content styled-scrollbar p-6 text-sm'>
+
+         <div className="settings__section__select mb-5">
+            <SelectField
+            label='Scraping Method'
+            options={scraperOptions}
+            selected={[settings.scraper_type || 'none']}
+            defaultLabel="Select Scraper"
+            updateField={(updatedTime:[string]) => updateSettings('scraper_type', updatedTime[0])}
+            multiple={false}
+            rounded={'rounded'}
+            minWidth={220}
+            />
+         </div>
+         {settings.scraper_type !== 'none' && settings.scraper_type !== 'proxy' && (
+            <div className="settings__section__secret mb-5">
+               <SecretField
+               label='Scraper API Key or Token'
+               placeholder={'API Key/Token'}
+               value={settings?.scaping_api || ''}
+               hasError={settingsError?.type === 'no_api_key'}
+               onChange={(value:string) => updateSettings('scaping_api', value)}
+               />
+            </div>
+         )}
+         {settings.scraper_type === 'proxy' && (
+            <div className="settings__section__input mb-5">
+               <label className={labelStyle}>Proxy List</label>
+               <textarea
+                  className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
+                  focus:outline-none min-h-[160px] focus:border-blue-200 
+                  ${settingsError?.type === 'no_email' ? ' border-red-400 focus:border-red-400' : ''} `}
+                  value={settings?.proxy}
+                  placeholder={'http://122.123.22.45:5049\nhttps://user:password@122.123.22.45:5049'}
+                  onChange={(event) => updateSettings('proxy', event.target.value)}
+               />
+            </div>
+         )}
+         {settings.scraper_type !== 'none' && (
+            <div className="settings__section__input mb-5">
+               <SelectField
+                  label='Scraping Frequency'
+                  multiple={false}
+                  selected={[settings?.scrape_interval || 'daily']}
+                  options={scrapingOptions}
+                  defaultLabel={'Notification Settings'}
+                  updateField={(updated:string[]) => updated[0] && updateSettings('scrape_interval', updated[0])}
+                  rounded='rounded'
+                  maxHeight={48}
+                  minWidth={220}
+               />
+               <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
+            </div>
+         )}
+            <div className="settings__section__input mb-5">
+               <SelectField
+                  label='keyword Scrape Delay'
+                  multiple={false}
+                  selected={[settings?.scrape_delay || '0']}
+                  options={delayOptions}
+                  defaultLabel={'Delay Settings'}
+                  updateField={(updated:string[]) => updated[0] && updateSettings('scrape_delay', updated[0])}
+                  rounded='rounded'
+                  maxHeight={48}
+                  minWidth={220}
+               />
+               <small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
+            </div>
+            <div className="settings__section__input mb-5">
+               <ToggleField
+               label='Auto Retry Failed Keyword Scrape'
+               value={!!settings?.scrape_retry }
+               onChange={(val) => updateSettings('scrape_retry', val)}
+               />
+            </div>
+            {settings?.scrape_retry && (settings.failed_queue?.length || 0) > 0 && (
+               <div className="settings__section__input mb-5">
+                  <label className={labelStyle}>Clear Failed Retry Queue</label>
+                  <button
+                  onClick={() => clearFailedMutate()}
+                  className=' py-3 px-5 w-full rounded cursor-pointer bg-gray-100 text-gray-800
+                  font-semibold text-sm hover:bg-gray-200'>
+                     {clearingQueue && <Icon type="loading" size={14} />} Clear Failed Queue
+                       ({settings.failed_queue?.length || 0} Keywords)
+                  </button>
+               </div>
+            )}
+      </div>
+   </div>
+   );
+};
+
+export default ScraperSettings;

components/settings/SearchConsoleSettings.tsx

@@ -0,0 +1,48 @@
+import React from 'react';
+import InputField from '../common/InputField';
+
+type SearchConsoleSettingsProps = {
+   settings: SettingsType,
+   settingsError: null | {
+      type: string,
+      msg: string
+   },
+   updateSettings: Function,
+}
+
+const SearchConsoleSettings = ({ settings, settingsError, updateSettings }:SearchConsoleSettingsProps) => {
+   return (
+      <div>
+      <div>
+
+         {/* <div className="settings__section__input mb-5">
+            <ToggleField
+            label='Enable Goolge Search Console'
+            value={settings?.scrape_retry ? 'true' : '' }
+            onChange={(val) => updateSettings('scrape_retry', val)}
+            />
+         </div> */}
+         <div className="settings__section__input mb-4 flex justify-between items-center w-full">
+            <InputField
+            label='Search Console Client Email'
+            onChange={(client_email:string) => updateSettings('search_console_client_email', client_email)}
+            value={settings.search_console_client_email}
+            placeholder='myapp@appspot.gserviceaccount.com'
+            />
+         </div>
+         <div className="settings__section__input mb-4 flex flex-col justify-between items-center w-full">
+            <label className='mb-2 font-semibold block text-sm text-gray-700 capitalize w-full'>Search Console Private Key</label>
+            <textarea
+               className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
+               focus:outline-none h-[100px] focus:border-blue-200`}
+               value={settings.search_console_private_key}
+               placeholder={'-----BEGIN PRIVATE KEY-----/ssssaswdkihad....'}
+               onChange={(event) => updateSettings('search_console_private_key', event.target.value)}
+            />
+         </div>
+      </div>
+   </div>
+   );
+};
+
+export default SearchConsoleSettings;

components/settings/Settings.tsx

@@ -1,8 +1,11 @@
 import React, { useEffect, useState } from 'react';
-// import { useQuery } from 'react-query';
-import useUpdateSettings, { useFetchSettings } from '../../services/settings';
+import { Toaster } from 'react-hot-toast';
+import { useFetchSettings, useUpdateSettings } from '../../services/settings';
 import Icon from '../common/Icon';
-import SelectField, { SelectionOption } from '../common/SelectField';
+import NotificationSettings from './NotificationSettings';
+import ScraperSettings from './ScraperSettings';
+import useOnKey from '../../hooks/useOnKey';
+import IntegrationSettings from './IntegrationSettings';
 
 type SettingsProps = {
    closeSettings: Function,
@@ -14,8 +17,10 @@ type SettingsError = {
    msg: string
 }
 
-const defaultSettings = {
+export const defaultSettings: SettingsType = {
    scraper_type: 'none',
+   scrape_delay: 'none',
+   scrape_retry: false,
    notification_interval: 'daily',
    notification_email: '',
    smtp_server: '',
@@ -23,6 +28,11 @@ const defaultSettings = {
    smtp_username: '',
    smtp_password: '',
    notification_email_from: '',
+   notification_email_from_name: 'SerpBear',
+   search_console: true,
+   search_console_client_email: '',
+   search_console_private_key: '',
+   keywordsColumns: ['Best', 'History', 'Volume', 'Search Console'],
 };
 
 const Settings = ({ closeSettings }:SettingsProps) => {
@@ -31,6 +41,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
    const [settingsError, setSettingsError] = useState<SettingsError|null>(null);
    const { mutate: updateMutate, isLoading: isUpdating } = useUpdateSettings(() => console.log(''));
    const { data: appSettings, isLoading } = useFetchSettings();
+   useOnKey('Escape', closeSettings);
 
    useEffect(() => {
       if (appSettings && appSettings.settings) {
@@ -38,48 +49,32 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       }
    }, [appSettings]);
 
-   useEffect(() => {
-      const closeModalonEsc = (event:KeyboardEvent) => {
-         if (event.key === 'Escape') {
-            console.log(event.key);
-            closeSettings();
-         }
-      };
-      window.addEventListener('keydown', closeModalonEsc, false);
-      return () => {
-         window.removeEventListener('keydown', closeModalonEsc, false);
-      };
-   }, [closeSettings]);
-
    const closeOnBGClick = (e:React.SyntheticEvent) => {
       e.stopPropagation();
       e.nativeEvent.stopImmediatePropagation();
       if (e.target === e.currentTarget) { closeSettings(); }
    };
 
-   const updateSettings = (key: string, value:string|number) => {
+   const updateSettings = (key: string, value:string|number|boolean) => {
       setSettings({ ...settings, [key]: value });
    };
 
    const performUpdate = () => {
       let error: null|SettingsError = null;
-      if (settings.notification_interval !== 'never') {
+      const { notification_interval, notification_email, notification_email_from, scraper_type, smtp_port, smtp_server, scaping_api } = settings;
+      if (notification_interval !== 'never') {
          if (!settings.notification_email) {
             error = { type: 'no_email', msg: 'Insert a Valid Email address' };
          }
-         if (settings.notification_email
-            && (!settings.smtp_username || !settings.smtp_password || !settings.smtp_port || !settings.smtp_server
-               || !settings.notification_email_from)) {
+         if (notification_email && (!smtp_port || !smtp_server || !notification_email_from)) {
                let type = 'no_smtp_from';
-               if (!settings.smtp_password) { type = 'no_smtp_pass'; }
-               if (!settings.smtp_username) { type = 'no_smtp_user'; }
-               if (!settings.smtp_port) { type = 'no_smtp_port'; }
-               if (!settings.smtp_server) { type = 'no_smtp_server'; }
+               if (!smtp_port) { type = 'no_smtp_port'; }
+               if (!smtp_server) { type = 'no_smtp_server'; }
                error = { type, msg: 'Insert SMTP Server details that will be used to send the emails.' };
          }
       }
 
-      if (['scrapingant', 'scrapingrobot'].includes(settings.scraper_type) && !settings.scaping_api) {
+      if (scraper_type !== 'proxy' && scraper_type !== 'none' && !scaping_api) {
          error = { type: 'no_api_key', msg: 'Insert a Valid API Key or Token for the Scraper Service.' };
       }
 
@@ -89,189 +84,64 @@ const Settings = ({ closeSettings }:SettingsProps) => {
       } else {
          // Perform Update
          updateMutate(settings);
+         // If Scraper is updated, refresh the page.
+         if (appSettings.settings === 'none' && scraper_type !== 'none') {
+            window.location.reload();
+         }
       }
    };
 
-   const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
+   const tabStyle = `inline-block px-3 py-2 rounded-md  cursor-pointer text-xs lg:text-sm lg:mr-3 lg:px-4 select-none z-10
+   text-gray-600 border border-b-0 relative top-[1px] rounded-b-none`;
+   const tabStyleActive = 'bg-white text-blue-600 border-slate-200';
 
-   const notificationOptions: SelectionOption[] = [
-      { label: 'Daily', value: 'daily' },
-      { label: 'Weekly', value: 'weekly' },
-      { label: 'Monthly', value: 'monthly' },
-      { label: 'Never', value: 'never' },
-   ];
-   const scraperOptions: SelectionOption[] = [
-      { label: 'None', value: 'none' },
-      { label: 'Proxy', value: 'proxy' },
-      { label: 'ScrapingAnt.com', value: 'scrapingant' },
-      { label: 'ScrapingRobot.com', value: 'scrapingrobot' },
-      { label: 'serply.io', value: 'serply' },
-   ];
-
-   const tabStyle = 'inline-block px-4 py-1 rounded-full mr-3 cursor-pointer text-sm';
    return (
        <div className="settings fixed w-full h-screen top-0 left-0 z-50" onClick={closeOnBGClick}>
-            <div className="absolute w-full max-w-xs bg-white customShadow top-0 right-0 h-screen" data-loading={isLoading} >
+            <div className="absolute w-full max-w-md bg-white customShadow top-0 right-0 h-screen" data-loading={isLoading} >
                {isLoading && <div className='absolute flex content-center items-center h-full'><Icon type="loading" size={24} /></div>}
-               <div className='settings__header p-6 border-b border-b-slate-200 text-slate-500'>
+               <div className='settings__header px-5 py-4 text-slate-500'>
                   <h3 className=' text-black text-lg font-bold'>Settings</h3>
                   <button
-                  className=' absolute top-2 right-2 p-2 px-3 text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
+                  className=' absolute top-2 right-2 p-2 px- text-gray-400 hover:text-gray-700 transition-all hover:rotate-90'
                   onClick={() => closeSettings()}>
                      <Icon type='close' size={24} />
                   </button>
                </div>
-               <div className=' px-4 mt-4 '>
+               <div className='border border-slate-200 px-3 py-4 pb-0 border-l-0 border-r-0 bg-[#f8f9ff]'>
                   <ul>
                      <li
-                     className={`${tabStyle} ${currentTab === 'scraper' ? ' bg-blue-50 text-blue-600' : ''}`}
+                     className={`${tabStyle} ${currentTab === 'scraper' ? tabStyleActive : 'border-transparent '}`}
                      onClick={() => setCurrentTab('scraper')}>
-                        Scraper
+                       <Icon type='scraper' /> Scraper
                      </li>
                      <li
-                     className={`${tabStyle} ${currentTab === 'notification' ? ' bg-blue-50 text-blue-600' : ''}`}
+                     className={`${tabStyle} ${currentTab === 'notification' ? tabStyleActive : 'border-transparent'}`}
                      onClick={() => setCurrentTab('notification')}>
-                        Notification
+                        <Icon type='email' /> Notification
+                     </li>
+                     <li
+                     className={`${tabStyle} ${currentTab === 'integrations' ? tabStyleActive : 'border-transparent'}`}
+                     onClick={() => setCurrentTab('integrations')}>
+                       <Icon type='integration' size={14} /> Integrations
                      </li>
                   </ul>
                </div>
-               {currentTab === 'scraper' && (
-                  <div>
-                     <div className='settings__content styled-scrollbar p-6 text-sm'>
-
-                        <div className="settings__section__select mb-5">
-                           <label className={labelStyle}>Scraping Method</label>
-                           <SelectField
-                           options={scraperOptions}
-                           selected={[settings.scraper_type || 'none']}
-                           defaultLabel="Select Scraper"
-                           updateField={(updatedTime:[string]) => updateSettings('scraper_type', updatedTime[0])}
-                           multiple={false}
-                           rounded={'rounded'}
-                           minWidth={270}
-                           />
-                        </div>
-                        {['scrapingant', 'scrapingrobot', 'serply'].includes(settings.scraper_type) && (
-                           <div className="settings__section__input mr-3">
-                              <label className={labelStyle}>Scraper API Key or Token</label>
-                              <input
-                                 className={`w-full p-2 border border-gray-200 rounded mt-2 mb-3 focus:outline-none  focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_api_key' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 type="text"
-                                 value={settings?.scaping_api || ''}
-                                 placeholder={'API Key/Token'}
-                                 onChange={(event) => updateSettings('scaping_api', event.target.value)}
-                              />
-                           </div>
-                        )}
-                        {settings.scraper_type === 'proxy' && (
-                           <div className="settings__section__input mb-5">
-                              <label className={labelStyle}>Proxy List</label>
-                              <textarea
-                                 className={`w-full p-2 border border-gray-200 rounded mb-3 text-xs 
-                                 focus:outline-none min-h-[160px] focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_email' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 value={settings?.proxy}
-                                 placeholder={'http://122.123.22.45:5049\nhttps://user:password@122.123.22.45:5049'}
-                                 onChange={(event) => updateSettings('proxy', event.target.value)}
-                              />
-                           </div>
-                        )}
-                     </div>
-                  </div>
+               {currentTab === 'scraper' && settings && (
+                  <ScraperSettings settings={settings} updateSettings={updateSettings} settingsError={settingsError} />
                )}
 
-            {currentTab === 'notification' && (
-               <div>
-                  <div className='settings__content styled-scrollbar p-6 text-sm'>
-                     <div className="settings__section__input mb-5">
-                        <label className={labelStyle}>Notification Frequency</label>
-                        <SelectField
-                           multiple={false}
-                           selected={[settings.notification_interval]}
-                           options={notificationOptions}
-                           defaultLabel={'Notification Settings'}
-                           updateField={(updated:string[]) => updated[0] && updateSettings('notification_interval', updated[0])}
-                           rounded='rounded'
-                           maxHeight={48}
-                           minWidth={270}
-                        />
-                     </div>
-                     {settings.notification_interval !== 'never' && (
-                        <>
-                           <div className="settings__section__input mb-5">
-                              <label className={labelStyle}>Notification Emails</label>
-                              <input
-                                 className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_email' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 type="text"
-                                 value={settings?.notification_email}
-                                 placeholder={'test@gmail.com'}
-                                 onChange={(event) => updateSettings('notification_email', event.target.value)}
-                              />
-                           </div>
-                           <div className="settings__section__input mb-5">
-                              <label className={labelStyle}>SMTP Server</label>
-                              <input
-                                 className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_smtp_server' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 type="text"
-                                 value={settings?.smtp_server || ''}
-                                 onChange={(event) => updateSettings('smtp_server', event.target.value)}
-                              />
-                           </div>
-                           <div className="settings__section__input mb-5">
-                              <label className={labelStyle}>SMTP Port</label>
-                              <input
-                                 className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_smtp_port' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 type="text"
-                                 value={settings?.smtp_port || ''}
-                                 onChange={(event) => updateSettings('smtp_port', event.target.value)}
-                              />
-                           </div>
-                           <div className="settings__section__input mb-5">
-                              <label className={labelStyle}>SMTP Username</label>
-                              <input
-                                 className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_smtp_user' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 type="text"
-                                 value={settings?.smtp_username || ''}
-                                 onChange={(event) => updateSettings('smtp_username', event.target.value)}
-                              />
-                           </div>
-                           <div className="settings__section__input mb-5">
-                              <label className={labelStyle}>SMTP Password</label>
-                              <input
-                                 className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_smtp_pass' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 type="text"
-                                 value={settings?.smtp_password || ''}
-                                 onChange={(event) => updateSettings('smtp_password', event.target.value)}
-                              />
-                           </div>
-                           <div className="settings__section__input mb-5">
-                              <label className={labelStyle}>From Email Address</label>
-                              <input
-                                 className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none  focus:border-blue-200 
-                                 ${settingsError && settingsError.type === 'no_smtp_from' ? ' border-red-400 focus:border-red-400' : ''} `}
-                                 type="text"
-                                 value={settings?.notification_email_from || ''}
-                                 placeholder="no-reply@mydomain.com"
-                                 onChange={(event) => updateSettings('notification_email_from', event.target.value)}
-                              />
-                           </div>
-                        </>
-                     )}
-
-                     </div>
-                     {settingsError && (
-                        <div className='absolute w-full bottom-16  text-center p-3 bg-red-100 text-red-600 text-sm font-semibold'>
-                           {settingsError.msg}
-                        </div>
-                     )}
-               </div>
-            )}
+               {currentTab === 'notification' && settings && (
+                  <NotificationSettings settings={settings} updateSettings={updateSettings} settingsError={settingsError} />
+               )}
+               {currentTab === 'integrations' && settings && (
+                  <IntegrationSettings
+                  settings={settings}
+                  updateSettings={updateSettings}
+                  settingsError={settingsError}
+                  performUpdate={performUpdate}
+                  closeSettings={closeSettings}
+                   />
+               )}
                <div className=' border-t-[1px] border-gray-200 p-2 px-3'>
                   <button
                   onClick={() => performUpdate()}
@@ -280,6 +150,7 @@ const Settings = ({ closeSettings }:SettingsProps) => {
                   </button>
                </div>
             </div>
+            <Toaster position='bottom-center' containerClassName="react_toaster" />
        </div>
    );
 };

cron.js

@@ -1,7 +1,8 @@
+/* eslint-disable no-new */
 const Cryptr = require('cryptr');
 const { promises } = require('fs');
 const { readFile } = require('fs');
-const cron = require('node-cron');
+const { Cron } = require('croner');
 require('dotenv').config({ path: './.env.local' });
 
 const getAppSettings = async () => {
@@ -49,11 +50,14 @@ const generateCronTime = (interval) => {
    if (interval === 'daily') {
       cronTime = '0 0 0 * * *';
    }
+   if (interval === 'other_day') {
+      cronTime = '0 0 2-30/2 * *';
+   }
    if (interval === 'daily_morning') {
-      cronTime = '0 0 0 3 * *';
+      cronTime = '0 0 3 * * *';
    }
    if (interval === 'weekly') {
-      cronTime = '0 0 0 */7 * *';
+      cronTime = '0 0 * * 1';
    }
    if (interval === 'monthly') {
       cronTime = '0 0 1 * *'; // Run every first day of the month at 00:00(midnight)
@@ -63,51 +67,30 @@ const generateCronTime = (interval) => {
 };
 
 const runAppCronJobs = () => {
-   // RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
-   const scrapeCronTime = generateCronTime('daily');
-   cron.schedule(scrapeCronTime, () => {
-      // console.log('### Running Keyword Position Cron Job!');
-      const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
-      fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
-      .then((res) => res.json())
-      .then((data) => console.log(data))
-      .catch((err) => {
-         console.log('ERROR Making Daily Scraper Cron Request..');
-         console.log(err);
-      });
-   }, { scheduled: true });
-
-   // Run Failed scraping CRON (Every Hour)
-   const failedCronTime = generateCronTime('hourly');
-   cron.schedule(failedCronTime, () => {
-      // console.log('### Retrying Failed Scrapes...');
-
-      readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
-         if (data) {
-            const keywordsToRetry = data ? JSON.parse(data) : [];
-            if (keywordsToRetry.length > 0) {
-               const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
-               fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/refresh?id=${keywordsToRetry.join(',')}`, fetchOpts)
-               .then((res) => res.json())
-               .then((refreshedData) => console.log(refreshedData))
-               .catch((fetchErr) => {
-                  console.log('ERROR Making failed_queue Cron Request..');
-                  console.log(fetchErr);
-               });
-            }
-         } else {
-            console.log('ERROR Reading Failed Scrapes Queue File..', err);
-         }
-      });
-   }, { scheduled: true });
-
-   // RUN Email Notification CRON
    getAppSettings().then((settings) => {
+      // RUN SERP Scraping CRON (EveryDay at Midnight) 0 0 0 * *
+      const scrape_interval = settings.scrape_interval || 'daily';
+      if (scrape_interval !== 'never') {
+         const scrapeCronTime = generateCronTime(scrape_interval);
+         new Cron(scrapeCronTime, () => {
+            // console.log('### Running Keyword Position Cron Job!');
+            const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
+            fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/cron`, fetchOpts)
+            .then((res) => res.json())
+            // .then((data) =>{ console.log(data)})
+            .catch((err) => {
+               console.log('ERROR Making SERP Scraper Cron Request..');
+               console.log(err);
+            });
+         }, { scheduled: true });
+      }
+
+      // RUN Email Notification CRON
       const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval;
       if (notif_interval) {
          const cronTime = generateCronTime(notif_interval === 'daily' ? 'daily_morning' : notif_interval);
          if (cronTime) {
-            cron.schedule(cronTime, () => {
+            new Cron(cronTime, () => {
                // console.log('### Sending Notification Email...');
                const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
                fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/notify`, fetchOpts)
@@ -121,6 +104,49 @@ const runAppCronJobs = () => {
          }
       }
    });
+
+   // Run Failed scraping CRON (Every Hour)
+   const failedCronTime = generateCronTime('hourly');
+   new Cron(failedCronTime, () => {
+      // console.log('### Retrying Failed Scrapes...');
+
+      readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' }, (err, data) => {
+         if (data) {
+            try {
+               const keywordsToRetry = data ? JSON.parse(data) : [];
+               if (keywordsToRetry.length > 0) {
+                  const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
+                  fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/refresh?id=${keywordsToRetry.join(',')}`, fetchOpts)
+                  .then((res) => res.json())
+                  .then((refreshedData) => console.log(refreshedData))
+                  .catch((fetchErr) => {
+                     console.log('ERROR Making failed_queue Cron Request..');
+                     console.log(fetchErr);
+                  });
+               }
+            } catch (error) {
+               console.log('ERROR Reading Failed Scrapes Queue File..', error);
+            }
+         } else {
+            console.log('ERROR Reading Failed Scrapes Queue File..', err);
+         }
+      });
+   }, { scheduled: true });
+
+   // Run Google Search Console Scraper Daily
+   if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) {
+      const searchConsoleCRONTime = generateCronTime('daily');
+      new Cron(searchConsoleCRONTime, () => {
+         const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } };
+         fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts)
+         .then((res) => res.json())
+         .then((data) => console.log(data))
+         .catch((err) => {
+            console.log('ERROR Making Google Search Console Scraper Cron Request..');
+            console.log(err);
+         });
+      }, { scheduled: true });
+   }
 };
 
 runAppCronJobs();

database/config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  production: {
+    username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER,
+    password: process.env.PASSWORD,
+    database: 'sequelize',
+    host: 'database',
+    port: 3306,
+    dialect: 'sqlite',
+    storage: './data/database.sqlite',
+    dialectOptions: {
+      bigNumberStrings: true,
+    },
+  },
+};

database/database.ts

@@ -6,7 +6,7 @@ import Keyword from './models/keyword';
 const connection = new Sequelize({
    dialect: 'sqlite',
    host: '0.0.0.0',
-   username: process.env.USER,
+   username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER,
    password: process.env.PASSWORD,
    database: 'sequelize',
    dialectModule: sqlite3,

database/migrations/1707068556345-add-new-keyword-fields.js

@@ -0,0 +1,45 @@
+// Migration: Adds city, latlong and settings keyword to keyword table.
+
+// CLI Migration
+module.exports = {
+   up: async (queryInterface, Sequelize) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (!keywordTableDefinition.city) {
+                  await queryInterface.addColumn('keyword', 'city', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+               if (!keywordTableDefinition.latlong) {
+                  await queryInterface.addColumn('keyword', 'latlong', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+               if (!keywordTableDefinition.settings) {
+                  await queryInterface.addColumn('keyword', 'settings', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+   down: (queryInterface) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (keywordTableDefinition.city) {
+                  await queryInterface.removeColumn('keyword', 'city', { transaction: t });
+               }
+               if (keywordTableDefinition.latlong) {
+                  await queryInterface.removeColumn('keyword', 'latlong', { transaction: t });
+               }
+               if (keywordTableDefinition.latlong) {
+                  await queryInterface.removeColumn('keyword', 'settings', { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+ };

database/migrations/1707233039698-add-domain-searchconsole-field.js

@@ -0,0 +1,29 @@
+// Migration: Adds search_console field to domain table to assign search console property type, url and api.
+
+// CLI Migration
+module.exports = {
+   up: (queryInterface, Sequelize) => {
+     return queryInterface.sequelize.transaction(async (t) => {
+      try {
+         const domainTableDefinition = await queryInterface.describeTable('domain');
+         if (domainTableDefinition && !domainTableDefinition.search_console) {
+            await queryInterface.addColumn('domain', 'search_console', { type: Sequelize.DataTypes.STRING }, { transaction: t });
+         }
+      } catch (error) {
+         console.log('error :', error);
+      }
+     });
+   },
+   down: (queryInterface) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const domainTableDefinition = await queryInterface.describeTable('domain');
+            if (domainTableDefinition && domainTableDefinition.search_console) {
+               await queryInterface.removeColumn('domain', 'search_console', { transaction: t });
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+ };

database/migrations/1709217223856-add-keyword-volume-field copy.js

@@ -0,0 +1,35 @@
+// Migration: Adds volume field to the keyword table.
+
+// CLI Migration
+module.exports = {
+   up: async (queryInterface, Sequelize) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (!keywordTableDefinition.volume) {
+                  await queryInterface.addColumn('keyword', 'volume', {
+                      type: Sequelize.DataTypes.STRING, allowNull: false, defaultValue: 0,
+                  }, { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+   down: (queryInterface) => {
+      return queryInterface.sequelize.transaction(async (t) => {
+         try {
+            const keywordTableDefinition = await queryInterface.describeTable('keyword');
+            if (keywordTableDefinition) {
+               if (keywordTableDefinition.volume) {
+                  await queryInterface.removeColumn('keyword', 'volume', { transaction: t });
+               }
+            }
+         } catch (error) {
+            console.log('error :', error);
+         }
+      });
+   },
+};

database/models/domain.ts

@@ -38,6 +38,9 @@ class Domain extends Model {
 
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
    notification_emails!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true })
+   search_console!: string;
 }
 
 export default Domain;

database/models/keyword.ts

@@ -19,7 +19,13 @@ class Keyword extends Model {
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: 'US' })
    country!: string;
 
-   @Column({ type: DataType.STRING, allowNull: false })
+   @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
+   city!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
+   latlong!: string;
+
+   @Column({ type: DataType.STRING, allowNull: false, defaultValue: '{}' })
    domain!: string;
 
    // @ForeignKey(() => Domain)
@@ -41,6 +47,9 @@ class Keyword extends Model {
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) })
    history!: string;
 
+   @Column({ type: DataType.INTEGER, allowNull: false, defaultValue: 0 })
+   volume!: number;
+
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) })
    url!: string;
 
@@ -58,6 +67,9 @@ class Keyword extends Model {
 
    @Column({ type: DataType.STRING, allowNull: true, defaultValue: 'false' })
    lastUpdateError!: string;
+
+   @Column({ type: DataType.STRING, allowNull: true })
+   settings!: string;
 }
 
 export default Keyword;

email/email.html

@@ -259,13 +259,26 @@
          color: #fff;
          border-radius: 4px 4px 0 0;
       }
-      .mainhead td {
+      .mainhead td, .subhead td {
          padding: 15px;
       }
       .mainhead a{
          color: white; 
          text-decoration: none;
       }
+      .subhead {
+         background: #dee3ff;
+         color: #344dd7;
+         border-radius: 4px 4px 0 0;
+      }
+      .subhead a {
+         color: #344dd7;
+         text-decoration: none;
+      }
+      .console_table{
+         margin-top: 40px;
+         margin-bottom: 20px;
+      }
       .keyword_table td {
          padding: 10px 0;
       }
@@ -277,13 +290,16 @@
       .keyword td:nth-child(1){
          font-weight: bold;
       }
-      .keyword_table th:nth-child(2), .keyword_table th:nth-child(3), .keyword td:nth-child(2), .keyword td:nth-child(3){
+      .keyword_table th:nth-child(2), .keyword_table th:nth-child(3), .keyword td:nth-child(2), .keyword td:nth-child(3), .keyword_table--sc th:nth-child(4), .keyword_table--sc td:nth-child(4){
          text-align: center;
       }
       .keyword td:nth-child(3){
          font-size: 12px;
          color: #888;
       }
+      .keyword_table--sc td:nth-child(3){
+         color:inherit;
+      }
       .keyword svg {
          width: 15px;
       }
@@ -304,6 +320,9 @@
          vertical-align: middle;
          opacity: 0.6;
       }
+      .google_icon{
+         max-width: 13px;
+      }
       /* -------------------------------------
           RESPONSIVE AND MOBILE FRIENDLY STYLES
       ------------------------------------- */
@@ -423,6 +442,7 @@
                               <tr align="left">
                                  <th>Keyword</th>
                                  <th>Position</th>
+                                 <th>Best</th>
                                  <th>Updated</th>
                                </tr>
                                {{keywordsTable}}
@@ -437,6 +457,9 @@
 
             <!-- END MAIN CONTENT AREA -->
             </table>
+
+            {{SCStatsTable}}
+
             <!-- END CENTERED WHITE CONTAINER -->
 
             <!-- START FOOTER -->

entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+npx sequelize-cli db:migrate --env production
+exec "$@"

hooks/useIsMobile.tsx

@@ -0,0 +1,12 @@
+import { useEffect, useState } from 'react';
+
+const useIsMobile = () => {
+   const [isMobile, setIsMobile] = useState<boolean>(false);
+   useEffect(() => {
+      setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches));
+   }, []);
+
+   return [isMobile];
+};
+
+export default useIsMobile;

hooks/useOnKey.tsx

@@ -0,0 +1,17 @@
+import { useEffect } from 'react';
+
+const useOnKey = (key:string, onPress: Function) => {
+   useEffect(() => {
+      const closeModalonEsc = (event:KeyboardEvent) => {
+         if (event.key === key) {
+            onPress();
+         }
+      };
+      window.addEventListener('keydown', closeModalonEsc, false);
+      return () => {
+         window.removeEventListener('keydown', closeModalonEsc, false);
+      };
+   }, [key, onPress]);
+};
+
+export default useOnKey;

hooks/useWindowResize.tsx

@@ -0,0 +1,13 @@
+import { useEffect } from 'react';
+
+const useWindowResize = (onResize: () => void) => {
+   useEffect(() => {
+      onResize();
+      window.addEventListener('resize', onResize);
+      return () => {
+          window.removeEventListener('resize', onResize);
+      };
+   }, [onResize]);
+};
+
+export default useWindowResize;

jest.setup.js

@@ -1,11 +1,26 @@
 // eslint-disable-next-line no-unused-vars
 import 'isomorphic-fetch';
 import './styles/globals.css';
+import '@testing-library/jest-dom';
+import { enableFetchMocks } from 'jest-fetch-mock';
 // Optional: configure or set up a testing framework before each test.
 // If you delete this file, remove `setupFilesAfterEnv` from `jest.config.js`
 
 // Used for __tests__/testing-library.js
 // Learn more: https://github.com/testing-library/jest-dom
-import '@testing-library/jest-dom/extend-expect';
+
+window.matchMedia = (query) => ({
+   matches: false,
+   media: query,
+   onchange: null,
+   addListener: jest.fn(), // deprecated
+   removeListener: jest.fn(), // deprecated
+   addEventListener: jest.fn(),
+   removeEventListener: jest.fn(),
+   dispatchEvent: jest.fn(),
+});
 
 global.ResizeObserver = require('resize-observer-polyfill');
+
+// Enable Fetch Mocking
+enableFetchMocks();

next.config.js

@@ -1,8 +1,13 @@
 /** @type {import('next').NextConfig} */
+const { version } = require('./package.json');
+
 const nextConfig = {
   reactStrictMode: true,
   swcMinify: false,
   output: 'standalone',
+  publicRuntimeConfig: {
+   version,
+ },
 };
 
 module.exports = nextConfig;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serpbear",
-  "version": "0.1.7",
+  "version": "2.0.7",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -9,66 +9,80 @@
     "cron": "node cron.js",
     "start:all": "concurrently npm:start npm:cron",
     "lint": "next lint",
+    "lint:css": "stylelint styles/*.css",
     "test": "jest --watch --verbose",
     "test:ci": "jest --ci",
     "test:cv": "jest --coverage --coverageDirectory='coverage'",
+    "db:migrate": "sequelize-cli db:migrate --env production",
+    "db:revert": "sequelize-cli db:migrate:undo --env production",
     "release": "standard-version"
   },
   "dependencies": {
-    "@testing-library/react": "^13.4.0",
+    "@googleapis/searchconsole": "^1.0.5",
+    "@isaacs/ttlcache": "^1.4.1",
     "@types/react-transition-group": "^4.4.5",
-    "axios": "^1.1.3",
+    "axios": "^1.7.7",
     "axios-retry": "^3.3.1",
     "chart.js": "^3.9.1",
-    "cheerio": "^1.0.0-rc.12",
+    "cheerio": "^1.0.0",
     "concurrently": "^7.6.0",
     "cookies": "^0.8.0",
+    "croner": "^9.0.0",
     "cryptr": "^6.0.3",
     "dayjs": "^1.11.5",
     "dotenv": "^16.0.3",
+    "google-auth-library": "^9.6.3",
     "https-proxy-agent": "^5.0.1",
     "isomorphic-fetch": "^3.0.0",
-    "jsonwebtoken": "^8.5.1",
-    "msw": "^0.49.0",
-    "next": "12.3.1",
-    "node-cron": "^3.0.2",
-    "nodemailer": "^6.8.0",
+    "jsonwebtoken": "^9.0.2",
+    "next": "^12.3.4",
+    "nodemailer": "^6.9.9",
     "react": "18.2.0",
     "react-chartjs-2": "^4.3.1",
     "react-dom": "18.2.0",
     "react-hot-toast": "^2.4.0",
+    "react-markdown": "^9.0.1",
     "react-query": "^3.39.2",
     "react-timeago": "^7.1.0",
     "react-transition-group": "^4.4.5",
+    "react-window": "^1.8.8",
     "reflect-metadata": "^0.1.13",
-    "sequelize": "^6.25.2",
-    "sequelize-typescript": "^2.1.5",
-    "sqlite3": "^5.1.2"
+    "sequelize": "^6.34.0",
+    "sequelize-typescript": "^2.1.6",
+    "sqlite3": "^5.1.7",
+    "umzug": "^3.8.2"
   },
   "devDependencies": {
-    "@testing-library/jest-dom": "^5.16.5",
+    "@testing-library/jest-dom": "^6.6.3",
+    "@testing-library/react": "^14.0.0",
     "@types/cookies": "^0.7.7",
     "@types/cryptr": "^4.0.1",
     "@types/isomorphic-fetch": "^0.0.36",
+    "@types/jest": "^29.5.8",
     "@types/jsonwebtoken": "^8.5.9",
     "@types/node": "18.11.0",
     "@types/nodemailer": "^6.4.6",
-    "@types/react": "18.0.21",
-    "@types/react-dom": "18.0.6",
+    "@types/react": "18.2.0",
+    "@types/react-dom": "18.2.0",
     "@types/react-timeago": "^4.1.3",
+    "@types/react-window": "^1.8.5",
     "autoprefixer": "^10.4.12",
     "eslint": "8.25.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-config-next": "12.3.1",
-    "jest": "^29.3.1",
-    "jest-environment-jsdom": "^29.3.1",
-    "postcss": "^8.4.18",
+    "jest": "^29.7.0",
+    "jest-environment-jsdom": "^29.7.0",
+    "jest-fetch-mock": "^3.0.3",
+    "msw": "^2.6.4",
+    "next-router-mock": "^0.9.10",
+    "postcss": "^8.4.49",
     "prettier": "^2.7.1",
     "resize-observer-polyfill": "^1.5.1",
-    "sass": "^1.55.0",
+    "sass": "^1.80.7",
+    "sequelize-cli": "^6.6.2",
     "standard-version": "^9.5.0",
     "stylelint-config-standard": "^29.0.0",
-    "tailwindcss": "^3.1.8",
-    "typescript": "4.8.4"
+    "tailwindcss": "^3.4.14",
+    "typescript": "^4.8.4"
   }
 }

pages/_app.tsx

@@ -5,7 +5,13 @@ import { QueryClient, QueryClientProvider } from 'react-query';
 import { ReactQueryDevtools } from 'react-query/devtools';
 
 function MyApp({ Component, pageProps }: AppProps) {
-   const [queryClient] = React.useState(() => new QueryClient());
+   const [queryClient] = React.useState(() => new QueryClient({
+      defaultOptions: {
+        queries: {
+          refetchOnWindowFocus: false,
+        },
+      },
+    }));
    return <QueryClientProvider client={queryClient}>
             <Component {...pageProps} />
             <ReactQueryDevtools initialIsOpen={false} />

pages/api/adwords.ts

@@ -0,0 +1,101 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { OAuth2Client } from 'google-auth-library';
+import { readFile, writeFile } from 'fs/promises';
+import Cryptr from 'cryptr';
+import db from '../../database/database';
+import verifyUser from '../../utils/verifyUser';
+import { getAdwordsCredentials, getAdwordsKeywordIdeas } from '../../utils/adwords';
+
+type adwordsValidateResp = {
+   valid: boolean
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   await db.sync();
+   const authorized = verifyUser(req, res);
+   if (authorized !== 'authorized') {
+      return res.status(401).json({ error: authorized });
+   }
+   if (req.method === 'GET') {
+      return getAdwordsRefreshToken(req, res);
+   }
+   if (req.method === 'POST') {
+      return validateAdwordsIntegration(req, res);
+   }
+   return res.status(502).json({ error: 'Unrecognized Route.' });
+}
+
+const getAdwordsRefreshToken = async (req: NextApiRequest, res: NextApiResponse<string>) => {
+   try {
+      const code = (req.query.code as string);
+      const https = req.headers.host?.includes('localhost:') ? 'http://' : 'https://';
+      const redirectURL = `${https}${req.headers.host}/api/adwords`;
+
+      if (code) {
+         try {
+            const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
+            const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {};
+            const cryptr = new Cryptr(process.env.SECRET as string);
+            const adwords_client_id = settings.adwords_client_id ? cryptr.decrypt(settings.adwords_client_id) : '';
+            const adwords_client_secret = settings.adwords_client_secret ? cryptr.decrypt(settings.adwords_client_secret) : '';
+            const oAuth2Client = new OAuth2Client(adwords_client_id, adwords_client_secret, redirectURL);
+            const r = await oAuth2Client.getToken(code);
+            if (r?.tokens?.refresh_token) {
+               const adwords_refresh_token = cryptr.encrypt(r.tokens.refresh_token);
+               await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify({ ...settings, adwords_refresh_token }), { encoding: 'utf-8' });
+               return res.status(200).send('Google Ads Intergrated Successfully! You can close this window.');
+            }
+            return res.status(400).send('Error Getting the Google Ads Refresh Token. Please Try Again!');
+         } catch (error:any) {
+            let errorMsg = error?.response?.data?.error;
+            if (errorMsg.includes('redirect_uri_mismatch')) {
+               errorMsg += ` Redirected URL: ${redirectURL}`;
+            }
+            console.log('[Error] Getting Google Ads Refresh Token! Reason: ', errorMsg);
+            return res.status(400).send(`Error Saving the Google Ads Refresh Token ${errorMsg ? `. Details: ${errorMsg}` : ''}. Please Try Again!`);
+         }
+      } else {
+         return res.status(400).send('No Code Provided By Google. Please Try Again!');
+      }
+   } catch (error) {
+      console.log('[ERROR] Getting Google Ads Refresh Token: ', error);
+      return res.status(400).send('Error Getting Google Ads Refresh Token. Please Try Again!');
+   }
+};
+
+const validateAdwordsIntegration = async (req: NextApiRequest, res: NextApiResponse<adwordsValidateResp>) => {
+   const errMsg = 'Error Validating Google Ads Integration. Please make sure your provided data are correct!';
+   const { developer_token, account_id } = req.body;
+   if (!developer_token || !account_id) {
+      return res.status(400).json({ valid: false, error: 'Please Provide the Google Ads Developer Token and Test Account ID' });
+   }
+   try {
+      // Save the Adwords Developer Token & Google Ads Test Account ID in App Settings
+      const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
+      const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {};
+      const cryptr = new Cryptr(process.env.SECRET as string);
+      const adwords_developer_token = cryptr.encrypt(developer_token.trim());
+      const adwords_account_id = cryptr.encrypt(account_id.trim());
+      const securedSettings = { ...settings, adwords_developer_token, adwords_account_id };
+      await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify(securedSettings), { encoding: 'utf-8' });
+
+      // Make a test Request to Google Ads
+      const adwordsCreds = await getAdwordsCredentials();
+      const { client_id, client_secret, refresh_token } = adwordsCreds || {};
+      if (adwordsCreds && client_id && client_secret && developer_token && account_id && refresh_token) {
+         const keywords = await getAdwordsKeywordIdeas(
+            adwordsCreds,
+            { country: 'US', language: '1000', keywords: ['compress'], seedType: 'custom' },
+             true,
+         );
+         if (keywords && Array.isArray(keywords) && keywords.length > 0) {
+            return res.status(200).json({ valid: true });
+         }
+      }
+      return res.status(400).json({ valid: false, error: errMsg });
+   } catch (error) {
+      console.log('[ERROR] Validating Google Ads Integration: ', error);
+      return res.status(400).json({ valid: false, error: errMsg });
+   }
+};

pages/api/clearfailed.ts

@@ -0,0 +1,29 @@
+import { writeFile } from 'fs/promises';
+import type { NextApiRequest, NextApiResponse } from 'next';
+import verifyUser from '../../utils/verifyUser';
+
+type SettingsGetResponse = {
+   cleared?: boolean,
+   error?: string,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized !== 'authorized') {
+      return res.status(401).json({ error: authorized });
+   }
+   if (req.method === 'PUT') {
+      return clearFailedQueue(req, res);
+   }
+   return res.status(502).json({ error: 'Unrecognized Route.' });
+}
+
+const clearFailedQueue = async (req: NextApiRequest, res: NextApiResponse<SettingsGetResponse>) => {
+   try {
+      await writeFile(`${process.cwd()}/data/failed_queue.json`, JSON.stringify([]), { encoding: 'utf-8' });
+      return res.status(200).json({ cleared: true });
+   } catch (error) {
+      console.log('[ERROR] Cleraring Failed Queue File.', error);
+      return res.status(200).json({ error: 'Error Cleraring Failed Queue!' });
+   }
+};

pages/api/cron.ts

@@ -3,7 +3,7 @@ import db from '../../database/database';
 import Keyword from '../../database/models/keyword';
 import { getAppSettings } from './settings';
 import verifyUser from '../../utils/verifyUser';
-import { refreshAndUpdateKeywords } from './refresh';
+import refreshAndUpdateKeywords from '../../utils/refresh';
 
 type CRONRefreshRes = {
    started: boolean
@@ -35,7 +35,7 @@ const cronRefreshkeywords = async (req: NextApiRequest, res: NextApiResponse<CRO
 
       return res.status(200).json({ started: true });
    } catch (error) {
-      console.log('ERROR cronRefreshkeywords: ', error);
+      console.log('[ERROR] CRON Refreshing Keywords: ', error);
       return res.status(400).json({ started: false, error: 'CRON Error refreshing keywords!' });
    }
 };

pages/api/dbmigrate.ts

@@ -0,0 +1,53 @@
+import { Sequelize } from 'sequelize';
+import { Umzug, SequelizeStorage } from 'umzug';
+import type { NextApiRequest, NextApiResponse } from 'next';
+import db from '../../database/database';
+import verifyUser from '../../utils/verifyUser';
+
+type MigrationGetResponse = {
+   hasMigrations: boolean,
+}
+
+type MigrationPostResponse = {
+   migrated: boolean,
+   erroor?: string
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized === 'authorized' && req.method === 'GET') {
+      await db.sync();
+      return getMigrationStatus(req, res);
+   }
+   if (authorized === 'authorized' && req.method === 'POST') {
+      return migrateDatabase(req, res);
+   }
+   return res.status(401).json({ error: authorized });
+}
+
+const getMigrationStatus = async (req: NextApiRequest, res: NextApiResponse<MigrationGetResponse>) => {
+   const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false });
+   const umzug = new Umzug({
+      migrations: { glob: 'database/migrations/*.js' },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: undefined,
+   });
+   const migrations = await umzug.pending();
+   // console.log('migrations :', migrations);
+   // const migrationsExceuted = await umzug.executed();
+   return res.status(200).json({ hasMigrations: migrations.length > 0 });
+};
+
+const migrateDatabase = async (req: NextApiRequest, res: NextApiResponse<MigrationPostResponse>) => {
+   const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false });
+   const umzug = new Umzug({
+      migrations: { glob: 'database/migrations/*.js' },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: undefined,
+   });
+   const migrations = await umzug.up();
+   console.log('[Updated] migrations :', migrations);
+   return res.status(200).json({ migrated: true });
+};

pages/api/domain.ts

@@ -0,0 +1,48 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import Cryptr from 'cryptr';
+import db from '../../database/database';
+import Domain from '../../database/models/domain';
+import verifyUser from '../../utils/verifyUser';
+
+type DomainGetResponse = {
+   domain?: DomainType | null
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   const authorized = verifyUser(req, res);
+   if (authorized === 'authorized' && req.method === 'GET') {
+      await db.sync();
+      return getDomain(req, res);
+   }
+   return res.status(401).json({ error: authorized });
+}
+
+const getDomain = async (req: NextApiRequest, res: NextApiResponse<DomainGetResponse>) => {
+   if (!req.query.domain && typeof req.query.domain !== 'string') {
+       return res.status(400).json({ error: 'Domain Name is Required!' });
+   }
+
+   try {
+      const query = { domain: req.query.domain as string };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const parsedDomain = foundDomain?.get({ plain: true }) || false;
+
+      if (parsedDomain && parsedDomain.search_console) {
+         try {
+            const cryptr = new Cryptr(process.env.SECRET as string);
+            const scData = JSON.parse(parsedDomain.search_console);
+            scData.client_email = scData.client_email ? cryptr.decrypt(scData.client_email) : '';
+            scData.private_key = scData.private_key ? cryptr.decrypt(scData.private_key) : '';
+            parsedDomain.search_console = JSON.stringify(scData);
+         } catch (error) {
+            console.log('[Error] Parsing Search Console Keys.');
+         }
+      }
+
+      return res.status(200).json({ domain: parsedDomain });
+   } catch (error) {
+      console.log('[ERROR] Getting Domain: ', error);
+      return res.status(400).json({ error: 'Error Loading Domain' });
+   }
+};

pages/api/domains.ts

@@ -1,22 +1,26 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
+import Cryptr from 'cryptr';
 import db from '../../database/database';
 import Domain from '../../database/models/domain';
 import Keyword from '../../database/models/keyword';
+import getdomainStats from '../../utils/domains';
 import verifyUser from '../../utils/verifyUser';
+import { checkSerchConsoleIntegration, removeLocalSCData } from '../../utils/searchConsole';
 
 type DomainsGetRes = {
-   domains: Domain[]
+   domains: DomainType[]
    error?: string|null,
 }
 
 type DomainsAddResponse = {
-   domain: Domain|null,
+   domains: DomainType[]|null,
    error?: string|null,
 }
 
 type DomainsDeleteRes = {
    domainRemoved: number,
    keywordsRemoved: number,
+   SCDataRemoved: boolean,
    error?: string|null,
 }
 
@@ -47,49 +51,62 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 }
 
 export const getDomains = async (req: NextApiRequest, res: NextApiResponse<DomainsGetRes>) => {
+   const withStats = !!req?.query?.withstats;
    try {
       const allDomains: Domain[] = await Domain.findAll();
-      return res.status(200).json({ domains: allDomains });
+      const formattedDomains: DomainType[] = allDomains.map((el) => {
+         const domainItem:DomainType = el.get({ plain: true });
+         const scData = domainItem?.search_console ? JSON.parse(domainItem.search_console) : {};
+         const { client_email, private_key } = scData;
+         const searchConsoleData = scData ? { ...scData, client_email: client_email ? 'true' : '', private_key: private_key ? 'true' : '' } : {};
+         return { ...domainItem, search_console: JSON.stringify(searchConsoleData) };
+      });
+      const theDomains: DomainType[] = withStats ? await getdomainStats(formattedDomains) : allDomains;
+      return res.status(200).json({ domains: theDomains });
    } catch (error) {
       return res.status(400).json({ domains: [], error: 'Error Getting Domains.' });
    }
 };
 
-export const addDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsAddResponse>) => {
-   if (!req.body.domain) {
-      return res.status(400).json({ domain: null, error: 'Error Adding Domain.' });
-   }
-   const { domain } = req.body || {};
-   const domainData = {
-      domain: domain.trim(),
-      slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-'),
-      lastUpdated: new Date().toJSON(),
-      added: new Date().toJSON(),
-   };
+const addDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsAddResponse>) => {
+   const { domains } = req.body;
+   if (domains && Array.isArray(domains) && domains.length > 0) {
+      const domainsToAdd: any = [];
 
-   try {
-      const addedDomain = await Domain.create(domainData);
-      return res.status(201).json({ domain: addedDomain });
-   } catch (error) {
-      return res.status(400).json({ domain: null, error: 'Error Adding Domain.' });
+      domains.forEach((domain: string) => {
+         domainsToAdd.push({
+            domain: domain.trim(),
+            slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-').replaceAll('/', '-'),
+            lastUpdated: new Date().toJSON(),
+            added: new Date().toJSON(),
+         });
+      });
+      try {
+         const newDomains:Domain[] = await Domain.bulkCreate(domainsToAdd);
+         const formattedDomains = newDomains.map((el) => el.get({ plain: true }));
+         return res.status(201).json({ domains: formattedDomains });
+      } catch (error) {
+         console.log('[ERROR] Adding New Domain ', error);
+         return res.status(400).json({ domains: [], error: 'Error Adding Domain.' });
+      }
+   } else {
+      return res.status(400).json({ domains: [], error: 'Necessary data missing.' });
    }
 };
 
 export const deleteDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsDeleteRes>) => {
    if (!req.query.domain && typeof req.query.domain !== 'string') {
-      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, error: 'Domain is Required!' });
+      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, SCDataRemoved: false, error: 'Domain is Required!' });
    }
    try {
       const { domain } = req.query || {};
       const removedDomCount: number = await Domain.destroy({ where: { domain } });
       const removedKeywordCount: number = await Keyword.destroy({ where: { domain } });
-      return res.status(200).json({
-            domainRemoved: removedDomCount,
-            keywordsRemoved: removedKeywordCount,
-         });
+      const SCDataRemoved = await removeLocalSCData(domain as string);
+      return res.status(200).json({ domainRemoved: removedDomCount, keywordsRemoved: removedKeywordCount, SCDataRemoved });
    } catch (error) {
-      console.log('##### Delete Domain Error: ', error);
-      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, error: 'Error Deleting Domain' });
+      console.log('[ERROR] Deleting Domain: ', req.query.domain, error);
+      return res.status(400).json({ domainRemoved: 0, keywordsRemoved: 0, SCDataRemoved: false, error: 'Error Deleting Domain' });
    }
 };
 
@@ -98,13 +115,28 @@ export const updateDomain = async (req: NextApiRequest, res: NextApiResponse<Dom
       return res.status(400).json({ domain: null, error: 'Domain is Required!' });
    }
    const { domain } = req.query || {};
-   const { notification_interval, notification_emails } = req.body;
+   const { notification_interval, notification_emails, search_console } = req.body as DomainSettings;
 
-   const domainToUpdate: Domain|null = await Domain.findOne({ where: { domain } });
-   if (domainToUpdate) {
-      domainToUpdate.set({ notification_interval, notification_emails });
-      await domainToUpdate.save();
+   try {
+      const domainToUpdate: Domain|null = await Domain.findOne({ where: { domain } });
+      // Validate Search Console API Data
+      if (domainToUpdate && search_console?.client_email && search_console?.private_key) {
+         const theDomainObj = domainToUpdate.get({ plain: true });
+         const isSearchConsoleAPIValid = await checkSerchConsoleIntegration({ ...theDomainObj, search_console: JSON.stringify(search_console) });
+         if (!isSearchConsoleAPIValid.isValid) {
+            return res.status(400).json({ domain: null, error: isSearchConsoleAPIValid.error });
+         }
+         const cryptr = new Cryptr(process.env.SECRET as string);
+         search_console.client_email = search_console.client_email ? cryptr.encrypt(search_console.client_email.trim()) : '';
+         search_console.private_key = search_console.private_key ? cryptr.encrypt(search_console.private_key.trim()) : '';
+      }
+      if (domainToUpdate) {
+         domainToUpdate.set({ notification_interval, notification_emails, search_console: JSON.stringify(search_console) });
+         await domainToUpdate.save();
+      }
+      return res.status(200).json({ domain: domainToUpdate });
+   } catch (error) {
+      console.log('[ERROR] Updating Domain: ', req.query.domain, error);
+      return res.status(400).json({ domain: null, error: 'Error Updating Domain. An Unknown Error Occured.' });
    }
-
-   return res.status(200).json({ domain: domainToUpdate });
 };

pages/api/ideas.ts

@@ -0,0 +1,111 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import db from '../../database/database';
+import verifyUser from '../../utils/verifyUser';
+import {
+   KeywordIdeasDatabase, getAdwordsCredentials, getAdwordsKeywordIdeas, getLocalKeywordIdeas, updateLocalKeywordIdeas,
+} from '../../utils/adwords';
+
+type keywordsIdeasUpdateResp = {
+   keywords: IdeaKeyword[],
+   error?: string|null,
+}
+
+type keywordsIdeasGetResp = {
+   data: KeywordIdeasDatabase|null,
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   await db.sync();
+   const authorized = verifyUser(req, res);
+   if (authorized !== 'authorized') {
+      return res.status(401).json({ error: authorized });
+   }
+   if (req.method === 'GET') {
+      return getKeywordIdeas(req, res);
+   }
+   if (req.method === 'POST') {
+      return updateKeywordIdeas(req, res);
+   }
+   if (req.method === 'PUT') {
+      return favoriteKeywords(req, res);
+   }
+   return res.status(502).json({ error: 'Unrecognized Route.' });
+}
+
+const getKeywordIdeas = async (req: NextApiRequest, res: NextApiResponse<keywordsIdeasGetResp>) => {
+   try {
+      const domain = req.query.domain as string;
+      if (domain) {
+         const keywordsDatabase = await getLocalKeywordIdeas(domain);
+         // console.log('keywords :', keywordsDatabase);
+         if (keywordsDatabase) {
+            return res.status(200).json({ data: keywordsDatabase });
+         }
+      }
+      return res.status(400).json({ data: null, error: 'Error Loading Keyword Ideas.' });
+   } catch (error) {
+      console.log('[ERROR] Fetching Keyword Ideas: ', error);
+      return res.status(400).json({ data: null, error: 'Error Loading Keyword Ideas.' });
+   }
+};
+
+const updateKeywordIdeas = async (req: NextApiRequest, res: NextApiResponse<keywordsIdeasUpdateResp>) => {
+   const errMsg = 'Error Fetching Keywords. Please try again!';
+   const { keywords = [], country = 'US', language = '1000', domain = '', seedSCKeywords, seedCurrentKeywords, seedType } = req.body;
+
+   if (!country || !language) {
+      return res.status(400).json({ keywords: [], error: 'Error Fetching Keywords. Please Provide a Country and Language' });
+   }
+   if (seedType === 'custom' && (keywords.length === 0 && !seedSCKeywords && !seedCurrentKeywords)) {
+      return res.status(400).json({ keywords: [], error: 'Error Fetching Keywords. Please Provide one of these: keywords, url or domain' });
+   }
+   try {
+      const adwordsCreds = await getAdwordsCredentials();
+      const { client_id, client_secret, developer_token, account_id, refresh_token } = adwordsCreds || {};
+      if (adwordsCreds && client_id && client_secret && developer_token && account_id && refresh_token) {
+         const ideaOptions = { country, language, keywords, domain, seedSCKeywords, seedCurrentKeywords, seedType };
+         const keywordIdeas = await getAdwordsKeywordIdeas(adwordsCreds, ideaOptions);
+         if (keywordIdeas && Array.isArray(keywordIdeas) && keywordIdeas.length > 1) {
+            return res.status(200).json({ keywords: keywordIdeas });
+         }
+      }
+      return res.status(400).json({ keywords: [], error: errMsg });
+   } catch (error) {
+      console.log('[ERROR] Fetching Keyword Ideas: ', error);
+      return res.status(400).json({ keywords: [], error: errMsg });
+   }
+};
+
+const favoriteKeywords = async (req: NextApiRequest, res: NextApiResponse<keywordsIdeasUpdateResp>) => {
+   const errMsg = 'Error Favorating Keyword Idea. Please try again!';
+   const { keywordID = '', domain = '' } = req.body;
+
+   if (!keywordID || !domain) {
+      return res.status(400).json({ keywords: [], error: 'Missing Necessary data. Please provide both keywordID and domain values.' });
+   }
+
+   try {
+      const keywordsDatabase = await getLocalKeywordIdeas(domain);
+      if (keywordsDatabase && keywordsDatabase.keywords) {
+         const theKeyword = keywordsDatabase.keywords.find((kw) => kw.uid === keywordID);
+         const existingKeywords = keywordsDatabase.favorites || [];
+         const newFavorites = [...existingKeywords];
+         const existingKeywordIndex = newFavorites.findIndex((kw) => kw.uid === keywordID);
+         if (existingKeywordIndex > -1) {
+            newFavorites.splice(existingKeywordIndex, 1);
+         } else if (theKeyword) newFavorites.push(theKeyword);
+
+         const updated = await updateLocalKeywordIdeas(domain, { favorites: newFavorites });
+
+         if (updated) {
+            return res.status(200).json({ keywords: newFavorites, error: '' });
+         }
+      }
+
+      return res.status(400).json({ keywords: [], error: errMsg });
+   } catch (error) {
+      console.log('[ERROR] Favorating Keyword Idea: ', error);
+      return res.status(400).json({ keywords: [], error: errMsg });
+   }
+};

pages/api/insight.ts

@@ -0,0 +1,64 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import db from '../../database/database';
+import { getCountryInsight, getKeywordsInsight, getPagesInsight } from '../../utils/insight';
+import { fetchDomainSCData, getSearchConsoleApiInfo, readLocalSCData } from '../../utils/searchConsole';
+import verifyUser from '../../utils/verifyUser';
+import Domain from '../../database/models/domain';
+
+type SCInsightRes = {
+   data: InsightDataType | null,
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   await db.sync();
+   const authorized = verifyUser(req, res);
+   if (authorized !== 'authorized') {
+      return res.status(401).json({ error: authorized });
+   }
+   if (req.method === 'GET') {
+      return getDomainSearchConsoleInsight(req, res);
+   }
+   return res.status(502).json({ error: 'Unrecognized Route.' });
+}
+
+const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiResponse<SCInsightRes>) => {
+   if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' });
+   const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
+   const getInsightFromSCData = (localSCData: SCDomainDataType): InsightDataType => {
+      const { stats = [] } = localSCData;
+      const countries = getCountryInsight(localSCData);
+      const keywords = getKeywordsInsight(localSCData);
+      const pages = getPagesInsight(localSCData);
+      return { pages, keywords, countries, stats };
+   };
+
+   // First try and read the  Local SC Domain Data file.
+   const localSCData = await readLocalSCData(domainname);
+
+   if (localSCData) {
+      const oldFetchedDate = localSCData.lastFetched;
+      const fetchTimeDiff = new Date().getTime() - (oldFetchedDate ? new Date(oldFetchedDate as string).getTime() : 0);
+      if (localSCData.stats && localSCData.stats.length && fetchTimeDiff <= 86400000) {
+         const response = getInsightFromSCData(localSCData);
+         return res.status(200).json({ data: response });
+      }
+   }
+
+   // If the Local SC Domain Data file does not exist, fetch from Googel Search Console.
+   try {
+      const query = { domain: domainname };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
+      const scDomainAPI = await getSearchConsoleApiInfo(domainObj);
+      if (!(scDomainAPI.client_email && scDomainAPI.private_key)) {
+         return res.status(200).json({ data: null, error: 'Google Search Console is not Integrated.' });
+      }
+      const scData = await fetchDomainSCData(domainObj, scDomainAPI);
+      const response = getInsightFromSCData(scData);
+      return res.status(200).json({ data: response });
+   } catch (error) {
+      console.log('[ERROR] Getting Domain Insight: ', domainname, error);
+      return res.status(400).json({ data: null, error: 'Error Fetching Stats from Google Search Console.' });
+   }
+};

pages/api/keyword.ts

@@ -22,7 +22,6 @@ const getKeyword = async (req: NextApiRequest, res: NextApiResponse<KeywordGetRe
    if (!req.query.id && typeof req.query.id !== 'string') {
        return res.status(400).json({ error: 'Keyword ID is Required!' });
    }
-   console.log('KEYWORD: ', req.query.id);
 
    try {
       const query = { ID: parseInt((req.query.id as string), 10) };
@@ -31,7 +30,7 @@ const getKeyword = async (req: NextApiRequest, res: NextApiResponse<KeywordGetRe
       const keywords = pareseKeyword && pareseKeyword[0] ? pareseKeyword[0] : null;
       return res.status(200).json({ keyword: keywords });
    } catch (error) {
-      console.log(error);
+      console.log('[ERROR] Getting Keyword: ', error);
       return res.status(400).json({ error: 'Error Loading Keyword' });
    }
 };

pages/api/keywords.ts

@@ -2,10 +2,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { Op } from 'sequelize';
 import db from '../../database/database';
 import Keyword from '../../database/models/keyword';
-import { refreshAndUpdateKeywords } from './refresh';
 import { getAppSettings } from './settings';
 import verifyUser from '../../utils/verifyUser';
 import parseKeywords from '../../utils/parseKeywords';
+import { integrateKeywordSCData, readLocalSCData } from '../../utils/searchConsole';
+import refreshAndUpdateKeywords from '../../utils/refresh';
+import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords';
 
 type KeywordsGetResponse = {
    keywords?: KeywordType[],
@@ -44,12 +46,16 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
    if (!req.query.domain && typeof req.query.domain !== 'string') {
       return res.status(400).json({ error: 'Domain is Required!' });
    }
-   const domain = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
+   const settings = await getAppSettings();
+   const domain = (req.query.domain as string);
+   const integratedSC = process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL;
+   const { search_console_client_email, search_console_private_key } = settings;
+   const domainSCData = integratedSC || (search_console_client_email && search_console_private_key) ? await readLocalSCData(domain) : false;
 
    try {
       const allKeywords:Keyword[] = await Keyword.findAll({ where: { domain } });
       const keywords: KeywordType[] = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
-      const slimKeywords = keywords.map((keyword) => {
+      const processedKeywords = keywords.map((keyword) => {
          const historyArray = Object.keys(keyword.history).map((dateKey:string) => ({
             date: new Date(dateKey).getTime(),
             dateRaw: dateKey,
@@ -58,29 +64,32 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
          const historySorted = historyArray.sort((a, b) => a.date - b.date);
          const lastWeekHistory :KeywordHistory = {};
          historySorted.slice(-7).forEach((x:any) => { lastWeekHistory[x.dateRaw] = x.position; });
-         return { ...keyword, lastResult: [], history: lastWeekHistory };
+         const keywordWithSlimHistory = { ...keyword, lastResult: [], history: lastWeekHistory };
+         const finalKeyword = domainSCData ? integrateKeywordSCData(keywordWithSlimHistory, domainSCData) : keywordWithSlimHistory;
+         return finalKeyword;
       });
-      console.log('getKeywords: ', keywords.length);
-      return res.status(200).json({ keywords: slimKeywords });
+      return res.status(200).json({ keywords: processedKeywords });
    } catch (error) {
-      console.log(error);
+      console.log('[ERROR] Getting Domain Keywords for ', domain, error);
       return res.status(400).json({ error: 'Error Loading Keywords for this Domain.' });
    }
 };
 
 const addKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGetResponse>) => {
-   const { keywords, device, country, domain, tags } = req.body;
-   if (keywords && device && country) {
-      const keywordsArray = keywords.replaceAll('\n', ',').split(',').map((item:string) => item.trim());
-      const tagsArray = tags ? tags.split(',').map((item:string) => item.trim()) : [];
+   const { keywords } = req.body;
+   if (keywords && Array.isArray(keywords) && keywords.length > 0) {
+      // const keywordsArray = keywords.replaceAll('\n', ',').split(',').map((item:string) => item.trim());
       const keywordsToAdd: any = []; // QuickFIX for bug: https://github.com/sequelize/sequelize-typescript/issues/936
 
-      keywordsArray.forEach((keyword: string) => {
+      keywords.forEach((kwrd: KeywordAddPayload) => {
+         const { keyword, device, country, domain, tags, city } = kwrd;
+         const tagsArray = tags ? tags.split(',').map((item:string) => item.trim()) : [];
          const newKeyword = {
             keyword,
             device,
             domain,
             country,
+            city,
             position: 0,
             updating: true,
             history: JSON.stringify({}),
@@ -97,10 +106,23 @@ const addKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
          const newKeywords:Keyword[] = await Keyword.bulkCreate(keywordsToAdd);
          const formattedkeywords = newKeywords.map((el) => el.get({ plain: true }));
          const keywordsParsed: KeywordType[] = parseKeywords(formattedkeywords);
+
+         // Queue the SERP Scraping Process
          const settings = await getAppSettings();
-         refreshAndUpdateKeywords(newKeywords, settings); // Queue the SERP Scraping Process
+         refreshAndUpdateKeywords(newKeywords, settings);
+
+         // Update the Keyword Volume
+         const { adwords_account_id, adwords_client_id, adwords_client_secret, adwords_developer_token } = settings;
+         if (adwords_account_id && adwords_client_id && adwords_client_secret && adwords_developer_token) {
+            const keywordsVolumeData = await getKeywordsVolume(keywordsParsed);
+            if (keywordsVolumeData.volumes !== false) {
+               await updateKeywordsVolumeData(keywordsVolumeData.volumes);
+            }
+         }
+
          return res.status(201).json({ keywords: keywordsParsed });
       } catch (error) {
+         console.log('[ERROR] Adding New Keywords ', error);
          return res.status(400).json({ error: 'Could Not Add New Keyword!' });
       }
    } else {
@@ -120,6 +142,7 @@ const deleteKeywords = async (req: NextApiRequest, res: NextApiResponse<Keywords
       const removedKeywordCount: number = await Keyword.destroy(removeQuery);
       return res.status(200).json({ keywordsRemoved: removedKeywordCount });
    } catch (error) {
+      console.log('[ERROR] Removing Keyword. ', error);
       return res.status(400).json({ error: 'Could Not Remove Keyword!' });
    }
 };
@@ -146,17 +169,20 @@ const updateKeywords = async (req: NextApiRequest, res: NextApiResponse<Keywords
       }
       if (tags) {
          const tagsKeywordIDs = Object.keys(tags);
+         const multipleKeywords = tagsKeywordIDs.length > 1;
          for (const keywordID of tagsKeywordIDs) {
-            const response = await Keyword.findOne({ where: { ID: keywordID } });
-            if (response) {
-               await response.update({ tags: JSON.stringify(tags[keywordID]) });
+            const selectedKeyword = await Keyword.findOne({ where: { ID: keywordID } });
+            const currentTags = selectedKeyword && selectedKeyword.tags ? JSON.parse(selectedKeyword.tags) : [];
+            const mergedTags = Array.from(new Set([...currentTags, ...tags[keywordID]]));
+            if (selectedKeyword) {
+               await selectedKeyword.update({ tags: JSON.stringify(multipleKeywords ? mergedTags : tags[keywordID]) });
             }
          }
          return res.status(200).json({ keywords });
       }
       return res.status(400).json({ error: 'Invalid Payload!' });
    } catch (error) {
-      console.log('ERROR updateKeywords: ', error);
+      console.log('[ERROR] Updating Keyword. ', error);
       return res.status(200).json({ error: 'Error Updating keywords!' });
    }
 };

pages/api/login.ts

@@ -18,10 +18,11 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
    if (!req.body.username || !req.body.password) {
       return res.status(401).json({ error: 'Username Password Missing' });
    }
+   const userName = process.env.USER_NAME ? process.env.USER_NAME : process.env.USER;
 
-   if (req.body.username === process.env.USER
+   if (req.body.username === userName
       && req.body.password === process.env.PASSWORD && process.env.SECRET) {
-      const token = jwt.sign({ user: process.env.USER }, process.env.SECRET);
+      const token = jwt.sign({ user: userName }, process.env.SECRET);
       const cookies = new Cookies(req, res);
       const expireDate = new Date();
       const sessDuration = process.env.SESSION_DURATION;
@@ -30,7 +31,7 @@ const loginUser = async (req: NextApiRequest, res: NextApiResponse<loginResponse
       return res.status(200).json({ success: true, error: null });
    }
 
-   const error = req.body.username !== process.env.USER ? 'Incorrect Username' : 'Incorrect Password';
+   const error = req.body.username !== userName ? 'Incorrect Username' : 'Incorrect Password';
 
    return res.status(401).json({ success: false, error });
 };

pages/api/notify.ts

@@ -21,44 +21,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 }
 
 const notify = async (req: NextApiRequest, res: NextApiResponse<NotifyResponse>) => {
+   const reqDomain = req?.query?.domain as string || '';
    try {
       const settings = await getAppSettings();
-      const {
-          smtp_server = '',
-          smtp_port = '',
-          smtp_username = '',
-          smtp_password = '',
-          notification_email = '',
-          notification_email_from = '',
-         } = settings;
+      const { smtp_server = '', smtp_port = '', notification_email = '' } = settings;
 
-      if (!smtp_server || !smtp_port || !smtp_username || !smtp_password || !notification_email) {
+      if (!smtp_server || !smtp_port || !notification_email) {
          return res.status(401).json({ success: false, error: 'SMTP has not been setup properly!' });
       }
-      const fromEmail = `SerpBear <${notification_email_from || 'no-reply@serpbear.com'}>`;
-      const transporter = nodeMailer.createTransport({
-         host: smtp_server,
-         port: parseInt(smtp_port, 10),
-         auth: { user: smtp_username, pass: smtp_password },
-      });
 
-      const allDomains: Domain[] = await Domain.findAll();
-
-      if (allDomains && allDomains.length > 0) {
-         const domains = allDomains.map((el) => el.get({ plain: true }));
-         for (const domain of domains) {
-            if (domain.notification !== false) {
-               const query = { where: { domain: domain.domain } };
-               const domainKeywords:Keyword[] = await Keyword.findAll(query);
-               const keywordsArray = domainKeywords.map((el) => el.get({ plain: true }));
-               const keywords: KeywordType[] = parseKeywords(keywordsArray);
-               await transporter.sendMail({
-                  from: fromEmail,
-                  to: domain.notification_emails || notification_email,
-                  subject: `[${domain.domain}] Keyword Positions Update`,
-                  html: await generateEmail(domain.domain, keywords),
-              });
-              // console.log(JSON.stringify(result, null, 4));
+      if (reqDomain) {
+         const theDomain = await Domain.findOne({ where: { domain: reqDomain } });
+         if (theDomain) {
+            await sendNotificationEmail(theDomain, settings);
+         }
+      } else {
+         const allDomains: Domain[] = await Domain.findAll();
+         if (allDomains && allDomains.length > 0) {
+            const domains = allDomains.map((el) => el.get({ plain: true }));
+            for (const domain of domains) {
+               if (domain.notification !== false) {
+                  await sendNotificationEmail(domain, settings);
+               }
             }
          }
       }
@@ -69,3 +53,36 @@ const notify = async (req: NextApiRequest, res: NextApiResponse<NotifyResponse>)
       return res.status(401).json({ success: false, error: 'Error Sending Notification Email.' });
    }
 };
+
+const sendNotificationEmail = async (domain: Domain, settings: SettingsType) => {
+   const {
+      smtp_server = '',
+      smtp_port = '',
+      smtp_username = '',
+      smtp_password = '',
+      notification_email = '',
+      notification_email_from = '',
+      notification_email_from_name = 'SerpBear',
+     } = settings;
+
+   const fromEmail = `${notification_email_from_name} <${notification_email_from || 'no-reply@serpbear.com'}>`;
+   const mailerSettings:any = { host: smtp_server, port: parseInt(smtp_port, 10) };
+   if (smtp_username || smtp_password) {
+      mailerSettings.auth = {};
+      if (smtp_username) mailerSettings.auth.user = smtp_username;
+      if (smtp_password) mailerSettings.auth.pass = smtp_password;
+   }
+   const transporter = nodeMailer.createTransport(mailerSettings);
+   const domainName = domain.domain;
+   const query = { where: { domain: domainName } };
+   const domainKeywords:Keyword[] = await Keyword.findAll(query);
+   const keywordsArray = domainKeywords.map((el) => el.get({ plain: true }));
+   const keywords: KeywordType[] = parseKeywords(keywordsArray);
+   const emailHTML = await generateEmail(domainName, keywords, settings);
+   await transporter.sendMail({
+      from: fromEmail,
+      to: domain.notification_emails || notification_email,
+      subject: `[${domainName}] Keyword Positions Update`,
+      html: emailHTML,
+   }).catch((err:any) => console.log('[ERROR] Sending Notification Email for', domainName, err?.response || err));
+};

pages/api/refresh.ts

@@ -2,23 +2,36 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { Op } from 'sequelize';
 import db from '../../database/database';
 import Keyword from '../../database/models/keyword';
-import refreshKeywords from '../../utils/refresh';
+import refreshAndUpdateKeywords from '../../utils/refresh';
 import { getAppSettings } from './settings';
 import verifyUser from '../../utils/verifyUser';
 import parseKeywords from '../../utils/parseKeywords';
-import { removeFromRetryQueue, retryScrape } from '../../utils/scraper';
+import { scrapeKeywordFromGoogle } from '../../utils/scraper';
 
 type KeywordsRefreshRes = {
    keywords?: KeywordType[]
    error?: string|null,
 }
 
+type KeywordSearchResultRes = {
+   searchResult?: {
+      results: { title: string, url: string, position: number }[],
+      keyword: string,
+      position: number,
+      country: string,
+   },
+   error?: string|null,
+}
+
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
    await db.sync();
    const authorized = verifyUser(req, res);
    if (authorized !== 'authorized') {
       return res.status(401).json({ error: authorized });
    }
+   if (req.method === 'GET') {
+      return getKeywordSearchResults(req, res);
+   }
    if (req.method === 'POST') {
       return refresTheKeywords(req, res);
    }
@@ -64,56 +77,46 @@ const refresTheKeywords = async (req: NextApiRequest, res: NextApiResponse<Keywo
    }
 };
 
-export const refreshAndUpdateKeywords = async (initKeywords:Keyword[], settings:SettingsType) => {
-   const formattedKeywords = initKeywords.map((el) => el.get({ plain: true }));
-   const refreshed: any = await refreshKeywords(formattedKeywords, settings);
-   // const fetchKeywords = await refreshKeywords(initialKeywords.map( k=> k.keyword ));
-   const updatedKeywords: KeywordType[] = [];
-
-   for (const keywordRaw of initKeywords) {
-      const keywordPrased = parseKeywords([keywordRaw.get({ plain: true })]);
-      const keyword = keywordPrased[0];
-      const udpatedkeyword = refreshed.find((item:any) => item.ID && item.ID === keyword.ID);
-
-      if (udpatedkeyword && keyword) {
-         const newPos = udpatedkeyword.position;
-         const newPosition = newPos !== false ? newPos : keyword.position;
-         const { history } = keyword;
-         const theDate = new Date();
-         history[`${theDate.getFullYear()}-${theDate.getMonth() + 1}-${theDate.getDate()}`] = newPosition;
-
-         const updatedVal = {
-            position: newPosition,
-            updating: false,
-            url: udpatedkeyword.url,
-            lastResult: udpatedkeyword.result,
-            history,
-            lastUpdated: udpatedkeyword.error ? keyword.lastUpdated : theDate.toJSON(),
-            lastUpdateError: udpatedkeyword.error
-               ? JSON.stringify({ date: theDate.toJSON(), error: `${udpatedkeyword.error}`, scraper: settings.scraper_type })
-               : 'false',
-         };
-         updatedKeywords.push({ ...keyword, ...{ ...updatedVal, lastUpdateError: JSON.parse(updatedVal.lastUpdateError) } });
-
-         // If failed, Add to Retry Queue Cron
-         if (udpatedkeyword.error) {
-            await retryScrape(keyword.ID);
-         } else {
-            await removeFromRetryQueue(keyword.ID);
-         }
-
-         // Update the Keyword Position in Database
-         try {
-            await keywordRaw.update({
-               ...updatedVal,
-               lastResult: Array.isArray(udpatedkeyword.result) ? JSON.stringify(udpatedkeyword.result) : udpatedkeyword.result,
-               history: JSON.stringify(history),
-            });
-            console.log('[SUCCESS] Updating the Keyword: ', keyword.keyword);
-         } catch (error) {
-            console.log('[ERROR] Updating SERP for Keyword', keyword.keyword, error);
-         }
-      }
+const getKeywordSearchResults = async (req: NextApiRequest, res: NextApiResponse<KeywordSearchResultRes>) => {
+   if (!req.query.keyword || !req.query.country || !req.query.device) {
+      return res.status(400).json({ error: 'A Valid keyword, Country Code, and device is Required!' });
+   }
+   try {
+      const settings = await getAppSettings();
+      if (!settings || (settings && settings.scraper_type === 'never')) {
+         return res.status(400).json({ error: 'Scraper has not been set up yet.' });
+      }
+      const dummyKeyword:KeywordType = {
+         ID: 99999999999999,
+         keyword: req.query.keyword as string,
+         device: 'desktop',
+         country: req.query.country as string,
+         domain: '',
+         lastUpdated: '',
+         volume: 0,
+         added: '',
+         position: 111,
+         sticky: false,
+         history: {},
+         lastResult: [],
+         url: '',
+         tags: [],
+         updating: false,
+         lastUpdateError: false,
+      };
+      const scrapeResult = await scrapeKeywordFromGoogle(dummyKeyword, settings);
+      if (scrapeResult && !scrapeResult.error) {
+         const searchResult = {
+            results: scrapeResult.result,
+            keyword: scrapeResult.keyword,
+            position: scrapeResult.position !== 111 ? scrapeResult.position : 0,
+            country: req.query.country as string,
+         };
+         return res.status(200).json({ error: '', searchResult });
+      }
+      return res.status(400).json({ error: 'Error Scraping Search Results for the given keyword!' });
+   } catch (error) {
+      console.log('ERROR refresThehKeywords: ', error);
+      return res.status(400).json({ error: 'Error refreshing keywords!' });
    }
-   return updatedKeywords;
 };

pages/api/searchconsole.ts

@@ -0,0 +1,67 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import db from '../../database/database';
+import Domain from '../../database/models/domain';
+import { fetchDomainSCData, getSearchConsoleApiInfo, readLocalSCData } from '../../utils/searchConsole';
+import verifyUser from '../../utils/verifyUser';
+
+type searchConsoleRes = {
+   data: SCDomainDataType|null
+   error?: string|null,
+}
+
+type searchConsoleCRONRes = {
+   status: string,
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   await db.sync();
+   const authorized = verifyUser(req, res);
+   if (authorized !== 'authorized') {
+      return res.status(401).json({ error: authorized });
+   }
+   if (req.method === 'GET') {
+      return getDomainSearchConsoleData(req, res);
+   }
+   if (req.method === 'POST') {
+      return cronRefreshSearchConsoleData(req, res);
+   }
+   return res.status(502).json({ error: 'Unrecognized Route.' });
+}
+
+const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse<searchConsoleRes>) => {
+   if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' });
+   const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
+   const localSCData = await readLocalSCData(domainname);
+   if (localSCData && localSCData.thirtyDays && localSCData.thirtyDays.length) {
+      return res.status(200).json({ data: localSCData });
+   }
+   try {
+      const query = { domain: domainname };
+      const foundDomain:Domain| null = await Domain.findOne({ where: query });
+      const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
+      const scDomainAPI = await getSearchConsoleApiInfo(domainObj);
+      if (!(scDomainAPI.client_email && scDomainAPI.private_key)) {
+         return res.status(200).json({ data: null, error: 'Google Search Console is not Integrated.' });
+      }
+      const scData = await fetchDomainSCData(domainObj, scDomainAPI);
+      return res.status(200).json({ data: scData });
+   } catch (error) {
+      console.log('[ERROR] Getting Search Console Data for: ', domainname, error);
+      return res.status(400).json({ data: null, error: 'Error Fetching Data from Google Search Console.' });
+   }
+};
+
+const cronRefreshSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse<searchConsoleCRONRes>) => {
+   try {
+      const allDomainsRaw = await Domain.findAll();
+      const Domains: DomainType[] = allDomainsRaw.map((el) => el.get({ plain: true }));
+      for (const domain of Domains) {
+         await fetchDomainSCData(domain);
+      }
+      return res.status(200).json({ status: 'completed' });
+   } catch (error) {
+      console.log('[ERROR] CRON Updating Search Console Data. ', error);
+      return res.status(400).json({ status: 'failed', error: 'Error Fetching Data from Google Search Console.' });
+   }
+};

pages/api/settings.ts

@@ -1,7 +1,9 @@
+import { writeFile, readFile } from 'fs/promises';
 import type { NextApiRequest, NextApiResponse } from 'next';
 import Cryptr from 'cryptr';
-import { writeFile, readFile } from 'fs/promises';
+import getConfig from 'next/config';
 import verifyUser from '../../utils/verifyUser';
+import allScrapers from '../../scrapers/index';
 
 type SettingsGetResponse = {
    settings?: object | null,
@@ -25,7 +27,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 const getSettings = async (req: NextApiRequest, res: NextApiResponse<SettingsGetResponse>) => {
    const settings = await getAppSettings();
    if (settings) {
-      return res.status(200).json({ settings });
+      const { publicRuntimeConfig } = getConfig();
+      const version = publicRuntimeConfig?.version;
+      return res.status(200).json({ settings: { ...settings, version } });
    }
    return res.status(400).json({ error: 'Error Loading Settings!' });
 };
@@ -38,21 +42,41 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
    }
    try {
       const cryptr = new Cryptr(process.env.SECRET as string);
-      const scaping_api = settings.scaping_api ? cryptr.encrypt(settings.scaping_api) : '';
-      const smtp_password = settings.smtp_password ? cryptr.encrypt(settings.smtp_password) : '';
-      const securedSettings = { ...settings, scaping_api, smtp_password };
+      const scaping_api = settings.scaping_api ? cryptr.encrypt(settings.scaping_api.trim()) : '';
+      const smtp_password = settings.smtp_password ? cryptr.encrypt(settings.smtp_password.trim()) : '';
+      const search_console_client_email = settings.search_console_client_email ? cryptr.encrypt(settings.search_console_client_email.trim()) : '';
+      const search_console_private_key = settings.search_console_private_key ? cryptr.encrypt(settings.search_console_private_key.trim()) : '';
+      const adwords_client_id = settings.adwords_client_id ? cryptr.encrypt(settings.adwords_client_id.trim()) : '';
+      const adwords_client_secret = settings.adwords_client_secret ? cryptr.encrypt(settings.adwords_client_secret.trim()) : '';
+      const adwords_developer_token = settings.adwords_developer_token ? cryptr.encrypt(settings.adwords_developer_token.trim()) : '';
+      const adwords_account_id = settings.adwords_account_id ? cryptr.encrypt(settings.adwords_account_id.trim()) : '';
+
+      const securedSettings = {
+         ...settings,
+         scaping_api,
+         smtp_password,
+         search_console_client_email,
+         search_console_private_key,
+         adwords_client_id,
+         adwords_client_secret,
+         adwords_developer_token,
+         adwords_account_id,
+      };
 
       await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify(securedSettings), { encoding: 'utf-8' });
       return res.status(200).json({ settings });
    } catch (error) {
-      console.log('ERROR updateSettings: ', error);
+      console.log('[ERROR] Updating App Settings. ', error);
       return res.status(200).json({ error: 'Error Updating Settings!' });
    }
 };
 
 export const getAppSettings = async () : Promise<SettingsType> => {
+   const screenshotAPIKey = process.env.SCREENSHOT_API || '69408-serpbear';
    try {
       const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
+      const failedQueueRaw = await readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' });
+      const failedQueue: string[] = failedQueueRaw ? JSON.parse(failedQueueRaw) : [];
       const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {};
       let decryptedSettings = settings;
 
@@ -60,25 +84,59 @@ export const getAppSettings = async () : Promise<SettingsType> => {
          const cryptr = new Cryptr(process.env.SECRET as string);
          const scaping_api = settings.scaping_api ? cryptr.decrypt(settings.scaping_api) : '';
          const smtp_password = settings.smtp_password ? cryptr.decrypt(settings.smtp_password) : '';
-         decryptedSettings = { ...settings, scaping_api, smtp_password };
+         const search_console_client_email = settings.search_console_client_email ? cryptr.decrypt(settings.search_console_client_email) : '';
+         const search_console_private_key = settings.search_console_private_key ? cryptr.decrypt(settings.search_console_private_key) : '';
+         const adwords_client_id = settings.adwords_client_id ? cryptr.decrypt(settings.adwords_client_id) : '';
+         const adwords_client_secret = settings.adwords_client_secret ? cryptr.decrypt(settings.adwords_client_secret) : '';
+         const adwords_developer_token = settings.adwords_developer_token ? cryptr.decrypt(settings.adwords_developer_token) : '';
+         const adwords_account_id = settings.adwords_account_id ? cryptr.decrypt(settings.adwords_account_id) : '';
+
+         decryptedSettings = {
+            ...settings,
+            scaping_api,
+            smtp_password,
+            search_console_client_email,
+            search_console_private_key,
+            search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL)
+            || !!(search_console_client_email && search_console_private_key),
+            available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id, allowsCity: !!scraper.allowsCity })),
+            failed_queue: failedQueue,
+            screenshot_key: screenshotAPIKey,
+            adwords_client_id,
+            adwords_client_secret,
+            adwords_developer_token,
+            adwords_account_id,
+         };
       } catch (error) {
          console.log('Error Decrypting Settings API Keys!');
       }
 
       return decryptedSettings;
    } catch (error) {
-      console.log(error);
-      const settings = {
+      console.log('[ERROR] Getting App Settings. ', error);
+      const settings: SettingsType = {
          scraper_type: 'none',
          notification_interval: 'never',
          notification_email: '',
          notification_email_from: '',
+         notification_email_from_name: 'SerpBear',
          smtp_server: '',
          smtp_port: '',
          smtp_username: '',
          smtp_password: '',
+         scrape_retry: false,
+         screenshot_key: screenshotAPIKey,
+         search_console: true,
+         search_console_client_email: '',
+         search_console_private_key: '',
+         keywordsColumns: ['Best', 'History', 'Volume', 'Search Console'],
+      };
+      const otherSettings = {
+         available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
+         failed_queue: [],
       };
       await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify(settings), { encoding: 'utf-8' });
-      return settings;
+      await writeFile(`${process.cwd()}/data/failed_queue.json`, JSON.stringify([]), { encoding: 'utf-8' });
+      return { ...settings, ...otherSettings };
    }
 };

pages/api/volume.ts

@@ -0,0 +1,66 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { Op } from 'sequelize';
+import db from '../../database/database';
+import Keyword from '../../database/models/keyword';
+import verifyUser from '../../utils/verifyUser';
+import parseKeywords from '../../utils/parseKeywords';
+import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords';
+
+type KeywordsRefreshRes = {
+   keywords?: KeywordType[]
+   error?: string|null,
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+   await db.sync();
+   const authorized = verifyUser(req, res);
+   if (authorized !== 'authorized') {
+      return res.status(401).json({ error: authorized });
+   }
+   if (req.method === 'POST') {
+      return updatekeywordVolume(req, res);
+   }
+   return res.status(502).json({ error: 'Unrecognized Route.' });
+}
+
+const updatekeywordVolume = async (req: NextApiRequest, res: NextApiResponse<KeywordsRefreshRes>) => {
+   const { keywords = [], domain = '', update = true } = req.body || {};
+   if (keywords.length === 0 && !domain) {
+      return res.status(400).json({ error: 'Please provide keyword Ids or a domain name.' });
+   }
+
+   try {
+      let keywordsToSend: KeywordType[] = [];
+      if (keywords.length > 0) {
+         const allKeywords:Keyword[] = await Keyword.findAll({ where: { ID: { [Op.in]: keywords } } });
+         keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
+      }
+      if (domain) {
+         const allDomain = domain === 'all';
+         const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
+         keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
+      }
+
+      if (keywordsToSend.length > 0) {
+         const keywordsVolumeData = await getKeywordsVolume(keywordsToSend);
+         if (keywordsVolumeData.error) {
+            return res.status(400).json({ keywords: [], error: keywordsVolumeData.error });
+         }
+         if (keywordsVolumeData.volumes !== false) {
+            if (update) {
+               const updated = await updateKeywordsVolumeData(keywordsVolumeData.volumes);
+               if (updated) {
+                  return res.status(200).json({ keywords });
+               }
+            }
+         } else {
+            return res.status(400).json({ error: 'Error Fetching Keywords Volume Data from Google Ads' });
+         }
+      }
+
+      return res.status(400).json({ keywords: [], error: 'Error Updating Keywords Volume data' });
+   } catch (error) {
+      console.log('[Error] updating keywords Volume Data: ', error);
+      return res.status(400).json({ error: 'Error Updating Keywords Volume data' });
+   }
+};

pages/domain/[slug]/index.tsx

@@ -1,9 +1,7 @@
-import React, { useEffect, useMemo, useState } from 'react';
+import React, { useMemo, useState } from 'react';
 import type { NextPage } from 'next';
 import Head from 'next/head';
 import { useRouter } from 'next/router';
-// import { useQuery } from 'react-query';
-// import toast from 'react-hot-toast';
 import { CSSTransition } from 'react-transition-group';
 import Sidebar from '../../../components/common/Sidebar';
 import TopBar from '../../../components/common/TopBar';
@@ -11,47 +9,47 @@ import DomainHeader from '../../../components/domains/DomainHeader';
 import KeywordsTable from '../../../components/keywords/KeywordsTable';
 import AddDomain from '../../../components/domains/AddDomain';
 import DomainSettings from '../../../components/domains/DomainSettings';
-import exportCSV from '../../../utils/exportcsv';
+import exportCSV from '../../../utils/client/exportcsv';
 import Settings from '../../../components/settings/Settings';
 import { useFetchDomains } from '../../../services/domains';
 import { useFetchKeywords } from '../../../services/keywords';
 import { useFetchSettings } from '../../../services/settings';
+import AddKeywords from '../../../components/keywords/AddKeywords';
+import Footer from '../../../components/common/Footer';
 
 const SingleDomain: NextPage = () => {
    const router = useRouter();
-   const [noScrapprtError, setNoScrapprtError] = useState(false);
    const [showAddKeywords, setShowAddKeywords] = useState(false);
    const [showAddDomain, setShowAddDomain] = useState(false);
    const [showDomainSettings, setShowDomainSettings] = useState(false);
    const [showSettings, setShowSettings] = useState(false);
    const [keywordSPollInterval, setKeywordSPollInterval] = useState<undefined|number>(undefined);
-   const { data: appSettings } = useFetchSettings();
+   const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings();
    const { data: domainsData } = useFetchDomains(router);
-   const { keywordsData, keywordsLoading } = useFetchKeywords(router, setKeywordSPollInterval, keywordSPollInterval);
+   const appSettings: SettingsType = appSettingsData?.settings || {};
+   const { scraper_type = '', available_scapers = [] } = appSettings;
+   const activeScraper = useMemo(() => available_scapers.find((scraper) => scraper.value === scraper_type), [scraper_type, available_scapers]);
 
-   const theDomains: Domain[] = (domainsData && domainsData.domains) || [];
-   const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
-
-   const activDomain: Domain|null = useMemo(() => {
-      let active:Domain|null = null;
+   const activDomain: DomainType|null = useMemo(() => {
+      let active:DomainType|null = null;
       if (domainsData?.domains && router.query?.slug) {
-         active = domainsData.domains.find((x:Domain) => x.slug === router.query.slug);
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
       }
       return active;
    }, [router.query.slug, domainsData]);
 
-   useEffect(() => {
-      console.log('appSettings.settings: ', appSettings && appSettings.settings);
-      if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) {
-         setNoScrapprtError(true);
-      }
-   }, [appSettings]);
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
 
-   // console.log('Domains Data:', router, activDomain, theKeywords);
+   const { keywordsData, keywordsLoading } = useFetchKeywords(router, activDomain?.domain || '', setKeywordSPollInterval, keywordSPollInterval);
+   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
+   const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
 
    return (
       <div className="Domain ">
-         {noScrapprtError && (
+         {((!scraper_type || (scraper_type === 'none')) && !isAppSettingsLoading) && (
                <div className=' p-3 bg-red-600 text-white text-sm text-center'>
                   A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
                </div>
@@ -64,39 +62,52 @@ const SingleDomain: NextPage = () => {
          <TopBar showSettings={() => setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} />
          <div className="flex w-full max-w-7xl mx-auto">
             <Sidebar domains={theDomains} showAddModal={() => setShowAddDomain(true)} />
-            <div className="domain_kewywords px-5 pt-10 lg:px-0 lg:pt-20 w-full">
+            <div className="domain_kewywords px-5 pt-10 lg:px-0 lg:pt-8 w-full">
                {activDomain && activDomain.domain
-               && <DomainHeader
+               ? <DomainHeader
                   domain={activDomain}
                   domains={theDomains}
                   showAddModal={setShowAddKeywords}
                   showSettingsModal={setShowDomainSettings}
                   exportCsv={() => exportCSV(theKeywords, activDomain.domain)}
-                  />}
+                  />
+                  : <div className='w-full lg:h-[100px]'></div>
+               }
                <KeywordsTable
                isLoading={keywordsLoading}
                domain={activDomain}
                keywords={theKeywords}
                showAddModal={showAddKeywords}
                setShowAddModal={setShowAddKeywords}
+               isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) || domainHasScAPI }
+               settings={appSettings}
                />
             </div>
          </div>
 
          <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
-            <AddDomain closeModal={() => setShowAddDomain(false)} />
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
          </CSSTransition>
 
          <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
             <DomainSettings
             domain={showDomainSettings && theDomains && activDomain && activDomain.domain ? activDomain : false}
-            domains={theDomains}
             closeModal={setShowDomainSettings}
             />
          </CSSTransition>
          <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
              <Settings closeSettings={() => setShowSettings(false)} />
          </CSSTransition>
+         <CSSTransition in={showAddKeywords} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddKeywords
+               domain={activDomain?.domain || ''}
+               scraperName={activeScraper?.label || ''}
+               keywords={theKeywords}
+               allowsCity={!!activeScraper?.allowsCity}
+               closeModal={() => setShowAddKeywords(false)}
+               />
+         </CSSTransition>
+         <Footer currentVersion={appSettings?.version ? appSettings.version : ''} />
       </div>
    );
 };

pages/domain/console/[slug]/index.tsx

@@ -0,0 +1,140 @@
+import React, { useMemo, useState } from 'react';
+import type { NextPage } from 'next';
+import Head from 'next/head';
+import { useRouter } from 'next/router';
+// import { useQuery } from 'react-query';
+// import toast from 'react-hot-toast';
+import { CSSTransition } from 'react-transition-group';
+import Sidebar from '../../../../components/common/Sidebar';
+import TopBar from '../../../../components/common/TopBar';
+import DomainHeader from '../../../../components/domains/DomainHeader';
+import AddDomain from '../../../../components/domains/AddDomain';
+import DomainSettings from '../../../../components/domains/DomainSettings';
+import exportCSV from '../../../../utils/client/exportcsv';
+import Settings from '../../../../components/settings/Settings';
+import { useFetchDomains } from '../../../../services/domains';
+import { useFetchSCKeywords } from '../../../../services/searchConsole';
+import SCKeywordsTable from '../../../../components/keywords/SCKeywordsTable';
+import { useFetchSettings } from '../../../../services/settings';
+import Footer from '../../../../components/common/Footer';
+
+const DiscoverPage: NextPage = () => {
+   const router = useRouter();
+   const [showDomainSettings, setShowDomainSettings] = useState(false);
+   const [showSettings, setShowSettings] = useState(false);
+   const [showAddDomain, setShowAddDomain] = useState(false);
+   const [scDateFilter, setSCDateFilter] = useState('thirtyDays');
+   const { data: appSettings } = useFetchSettings();
+   const { data: domainsData } = useFetchDomains(router);
+   const scConnected = !!(appSettings && appSettings?.settings?.search_console_integrated);
+   const { data: keywordsData, isLoading: keywordsLoading, isFetching } = useFetchSCKeywords(router, !!(domainsData?.domains?.length) && scConnected);
+
+   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
+   const theKeywords: SearchAnalyticsItem[] = useMemo(() => {
+      return keywordsData?.data && keywordsData.data[scDateFilter] ? keywordsData.data[scDateFilter] : [];
+   }, [keywordsData, scDateFilter]);
+
+   const theKeywordsCount = useMemo(() => {
+      return theKeywords.reduce<Map<string, number>>((r, o) => {
+         const key = `${o.device}-${o.country}-${o.keyword}`;
+         const item = r.get(key) || 0;
+         return r.set(key, item + 1);
+      }, new Map()) || [];
+   }, [theKeywords]);
+
+   const theKeywordsReduced : SearchAnalyticsItem[] = useMemo(() => {
+      return [...theKeywords.reduce<Map<string, SearchAnalyticsItem>>((r, o) => {
+         const key = `${o.device}-${o.country}-${o.keyword}`;
+         const item = r.get(key) || { ...o,
+            ...{
+            clicks: 0,
+            impressions: 0,
+            ctr: 0,
+            position: 0,
+            },
+         };
+         item.clicks += o.clicks;
+         item.impressions += o.impressions;
+         item.ctr = o.ctr + item.ctr;
+         item.position = o.position + item.position;
+         return r.set(key, item);
+      }, new Map()).values()];
+   }, [theKeywords]);
+
+   const theKeywordsGrouped : SearchAnalyticsItem[] = useMemo(() => {
+      return [...theKeywordsReduced.map<SearchAnalyticsItem>((o: SearchAnalyticsItem) => {
+         const key = `${o.device}-${o.country}-${o.keyword}`;
+         const count = theKeywordsCount?.get(key) || 0;
+         return { ...o,
+            ...{
+            ctr: Math.round((o.ctr / count) * 100) / 100,
+            position: Math.round(o.position / count),
+            },
+         };
+      })];
+   }, [theKeywordsReduced, theKeywordsCount]);
+
+   const activDomain: DomainType|null = useMemo(() => {
+      let active:DomainType|null = null;
+      if (domainsData?.domains && router.query?.slug) {
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
+      }
+      return active;
+   }, [router.query.slug, domainsData]);
+
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
+
+   return (
+      <div className="Domain ">
+         {activDomain && activDomain.domain
+         && <Head>
+               <title>{`${activDomain.domain} - SerpBear` } </title>
+            </Head>
+         }
+         <TopBar showSettings={() => setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} />
+         <div className="flex w-full max-w-7xl mx-auto">
+            <Sidebar domains={theDomains} showAddModal={() => setShowAddDomain(true)} />
+            <div className="domain_kewywords px-5 pt-10 lg:px-0 lg:pt-8 w-full">
+               {activDomain && activDomain.domain
+               ? <DomainHeader
+                  domain={activDomain}
+                  domains={theDomains}
+                  showAddModal={() => console.log('XXXXX')}
+                  showSettingsModal={setShowDomainSettings}
+                  exportCsv={() => exportCSV(theKeywordsGrouped, activDomain.domain, scDateFilter)}
+                  scFilter={scDateFilter}
+                  setScFilter={(item:string) => setSCDateFilter(item)}
+                  />
+                  : <div className='w-full lg:h-[100px]'></div>
+               }
+               <SCKeywordsTable
+               isLoading={keywordsLoading || isFetching}
+               domain={activDomain}
+               keywords={theKeywordsGrouped}
+               isConsoleIntegrated={scConnected || domainHasScAPI}
+               />
+            </div>
+         </div>
+
+         <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
+         </CSSTransition>
+
+         <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <DomainSettings
+            domain={showDomainSettings && theDomains && activDomain && activDomain.domain ? activDomain : false}
+            closeModal={setShowDomainSettings}
+            />
+         </CSSTransition>
+         <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
+             <Settings closeSettings={() => setShowSettings(false)} />
+         </CSSTransition>
+         <Footer currentVersion={appSettings?.settings?.version ? appSettings.settings.version : ''} />
+      </div>
+   );
+};
+
+export default DiscoverPage;

pages/domain/ideas/[slug]/index.tsx

@@ -0,0 +1,113 @@
+import React, { useMemo, useState } from 'react';
+import type { NextPage } from 'next';
+import Head from 'next/head';
+import { useRouter } from 'next/router';
+import { CSSTransition } from 'react-transition-group';
+import Sidebar from '../../../../components/common/Sidebar';
+import TopBar from '../../../../components/common/TopBar';
+import DomainHeader from '../../../../components/domains/DomainHeader';
+import AddDomain from '../../../../components/domains/AddDomain';
+import DomainSettings from '../../../../components/domains/DomainSettings';
+import { exportKeywordIdeas } from '../../../../utils/client/exportcsv';
+import Settings from '../../../../components/settings/Settings';
+import { useFetchDomains } from '../../../../services/domains';
+import { useFetchSettings } from '../../../../services/settings';
+import KeywordIdeasTable from '../../../../components/ideas/KeywordIdeasTable';
+import { useFetchKeywordIdeas } from '../../../../services/adwords';
+import KeywordIdeasUpdater from '../../../../components/ideas/KeywordIdeasUpdater';
+import Modal from '../../../../components/common/Modal';
+import Footer from '../../../../components/common/Footer';
+
+const DiscoverPage: NextPage = () => {
+   const router = useRouter();
+   const [showDomainSettings, setShowDomainSettings] = useState(false);
+   const [showSettings, setShowSettings] = useState(false);
+   const [showAddDomain, setShowAddDomain] = useState(false);
+   const [showUpdateModal, setShowUpdateModal] = useState(false);
+   const [showFavorites, setShowFavorites] = useState(false);
+
+   const { data: appSettings } = useFetchSettings();
+   const { data: domainsData } = useFetchDomains(router);
+   const adwordsConnected = !!(appSettings && appSettings?.settings?.adwords_refresh_token
+      && appSettings?.settings?.adwords_developer_token, appSettings?.settings?.adwords_account_id);
+   const searchConsoleConnected = !!(appSettings && appSettings?.settings?.search_console_integrated);
+   const { data: keywordIdeasData, isLoading: isLoadingIdeas, isError: errorLoadingIdeas } = useFetchKeywordIdeas(router, adwordsConnected);
+   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
+   const keywordIdeas:IdeaKeyword[] = keywordIdeasData?.data?.keywords || [];
+   const favorites:IdeaKeyword[] = keywordIdeasData?.data?.favorites || [];
+   const keywordIdeasSettings = keywordIdeasData?.data?.settings || undefined;
+
+   const activDomain: DomainType|null = useMemo(() => {
+      let active:DomainType|null = null;
+      if (domainsData?.domains && router.query?.slug) {
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
+      }
+      return active;
+   }, [router.query.slug, domainsData]);
+
+   return (
+      <div className="Domain ">
+         {activDomain && activDomain.domain
+         && <Head>
+               <title>{`${activDomain.domain} - Keyword Ideas` } </title>
+            </Head>
+         }
+         <TopBar showSettings={() => setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} />
+         <div className="flex w-full max-w-7xl mx-auto">
+            <Sidebar domains={theDomains} showAddModal={() => setShowAddDomain(true)} />
+            <div className="domain_kewywords px-5 pt-10 lg:px-0 lg:pt-8 w-full">
+               {activDomain && activDomain.domain ? (
+                  <DomainHeader
+                  domain={activDomain}
+                  domains={theDomains}
+                  showAddModal={() => console.log('XXXXX')}
+                  showSettingsModal={setShowDomainSettings}
+                  exportCsv={() => exportKeywordIdeas(showFavorites ? favorites : keywordIdeas, activDomain.domain)}
+                  showIdeaUpdateModal={() => setShowUpdateModal(true)}
+                  />
+               ) : <div className='w-full lg:h-[100px]'></div>}
+               <KeywordIdeasTable
+               isLoading={isLoadingIdeas}
+               noIdeasDatabase={errorLoadingIdeas}
+               domain={activDomain}
+               keywords={keywordIdeas}
+               favorites={favorites}
+               isAdwordsIntegrated={adwordsConnected}
+               showFavorites={showFavorites}
+               setShowFavorites={setShowFavorites}
+               />
+            </div>
+         </div>
+
+         <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
+         </CSSTransition>
+
+         <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <DomainSettings
+            domain={showDomainSettings && theDomains && activDomain && activDomain.domain ? activDomain : false}
+            closeModal={setShowDomainSettings}
+            />
+         </CSSTransition>
+
+         <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
+             <Settings closeSettings={() => setShowSettings(false)} />
+         </CSSTransition>
+
+         {showUpdateModal && activDomain?.domain && (
+            <Modal closeModal={() => setShowUpdateModal(false) } title={'Load Keyword Ideas from Google Ads'} verticalCenter={true}>
+               <KeywordIdeasUpdater
+               domain={activDomain}
+               onUpdate={() => setShowUpdateModal(false)}
+               settings={keywordIdeasSettings}
+               searchConsoleConnected={searchConsoleConnected}
+               adwordsConnected={adwordsConnected}
+               />
+            </Modal>
+         )}
+         <Footer currentVersion={appSettings?.settings?.version ? appSettings.settings.version : ''} />
+      </div>
+   );
+};
+
+export default DiscoverPage;

pages/domain/insight/[slug]/index.tsx

@@ -0,0 +1,98 @@
+import React, { useMemo, useState } from 'react';
+import type { NextPage } from 'next';
+import Head from 'next/head';
+import { useRouter } from 'next/router';
+// import { useQuery } from 'react-query';
+// import toast from 'react-hot-toast';
+import { CSSTransition } from 'react-transition-group';
+import Sidebar from '../../../../components/common/Sidebar';
+import TopBar from '../../../../components/common/TopBar';
+import DomainHeader from '../../../../components/domains/DomainHeader';
+import AddDomain from '../../../../components/domains/AddDomain';
+import DomainSettings from '../../../../components/domains/DomainSettings';
+import exportCSV from '../../../../utils/client/exportcsv';
+import Settings from '../../../../components/settings/Settings';
+import { useFetchDomains } from '../../../../services/domains';
+import { useFetchSCInsight } from '../../../../services/searchConsole';
+import SCInsight from '../../../../components/insight/Insight';
+import { useFetchSettings } from '../../../../services/settings';
+import Footer from '../../../../components/common/Footer';
+
+const InsightPage: NextPage = () => {
+   const router = useRouter();
+   const [showDomainSettings, setShowDomainSettings] = useState(false);
+   const [showSettings, setShowSettings] = useState(false);
+   const [showAddDomain, setShowAddDomain] = useState(false);
+   const [scDateFilter, setSCDateFilter] = useState('thirtyDays');
+   const { data: appSettings } = useFetchSettings();
+   const { data: domainsData } = useFetchDomains(router);
+   const scConnected = !!(appSettings && appSettings?.settings?.search_console_integrated);
+   const { data: insightData } = useFetchSCInsight(router, !!(domainsData?.domains?.length) && scConnected);
+
+   const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
+   const theInsight: InsightDataType = insightData && insightData.data ? insightData.data : {};
+
+   const activDomain: DomainType|null = useMemo(() => {
+      let active:DomainType|null = null;
+      if (domainsData?.domains && router.query?.slug) {
+         active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null;
+      }
+      return active;
+   }, [router.query.slug, domainsData]);
+
+   const domainHasScAPI = useMemo(() => {
+      const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {};
+      return !!(doaminSc?.client_email && doaminSc?.private_key);
+   }, [activDomain]);
+
+   return (
+      <div className="Domain ">
+         {activDomain && activDomain.domain
+         && <Head>
+               <title>{`${activDomain.domain} - SerpBear` } </title>
+            </Head>
+         }
+         <TopBar showSettings={() => setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} />
+         <div className="flex w-full max-w-7xl mx-auto">
+            <Sidebar domains={theDomains} showAddModal={() => setShowAddDomain(true)} />
+            <div className="domain_kewywords px-5 pt-10 lg:px-0 lg:pt-8 w-full">
+               {activDomain && activDomain.domain
+               ? <DomainHeader
+                  domain={activDomain}
+                  domains={theDomains}
+                  showAddModal={() => console.log('XXXXX')}
+                  showSettingsModal={setShowDomainSettings}
+                  exportCsv={() => exportCSV([], activDomain.domain, scDateFilter)}
+                  scFilter={scDateFilter}
+                  setScFilter={(item:string) => setSCDateFilter(item)}
+                  />
+                  : <div className='w-full lg:h-[100px]'></div>
+               }
+               <SCInsight
+               isLoading={false}
+               domain={activDomain}
+               insight={theInsight}
+               isConsoleIntegrated={scConnected || domainHasScAPI}
+               />
+            </div>
+         </div>
+
+         <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
+         </CSSTransition>
+
+         <CSSTransition in={showDomainSettings} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <DomainSettings
+            domain={showDomainSettings && theDomains && activDomain && activDomain.domain ? activDomain : false}
+            closeModal={setShowDomainSettings}
+            />
+         </CSSTransition>
+         <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
+             <Settings closeSettings={() => setShowSettings(false)} />
+         </CSSTransition>
+         <Footer currentVersion={appSettings?.settings?.version ? appSettings.settings.version : ''} />
+      </div>
+   );
+};
+
+export default InsightPage;

pages/domains/index.tsx

@@ -0,0 +1,148 @@
+import React, { useEffect, useMemo, useState } from 'react';
+import type { NextPage } from 'next';
+import Head from 'next/head';
+import { useRouter } from 'next/router';
+import { CSSTransition } from 'react-transition-group';
+import toast, { Toaster } from 'react-hot-toast';
+import TopBar from '../../components/common/TopBar';
+import AddDomain from '../../components/domains/AddDomain';
+import Settings from '../../components/settings/Settings';
+import { useCheckMigrationStatus, useFetchSettings } from '../../services/settings';
+import { fetchDomainScreenshot, useFetchDomains } from '../../services/domains';
+import DomainItem from '../../components/domains/DomainItem';
+import Icon from '../../components/common/Icon';
+import Footer from '../../components/common/Footer';
+
+type thumbImages = { [domain:string] : string }
+
+const Domains: NextPage = () => {
+   const router = useRouter();
+   // const [noScrapprtError, setNoScrapprtError] = useState(false);
+   const [showSettings, setShowSettings] = useState(false);
+   const [showAddDomain, setShowAddDomain] = useState(false);
+   const [domainThumbs, setDomainThumbs] = useState<thumbImages>({});
+   const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings();
+   const { data: domainsData, isLoading } = useFetchDomains(router, true);
+   const { data: migrationStatus } = useCheckMigrationStatus();
+
+   const appSettings:SettingsType = appSettingsData?.settings || {};
+   const { scraper_type = '' } = appSettings;
+
+   const totalKeywords = useMemo(() => {
+      let keywords = 0;
+      if (domainsData?.domains) {
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            keywords += domain?.keywordCount || 0;
+         });
+      }
+      return keywords;
+   }, [domainsData]);
+
+   const domainSCAPiObj = useMemo(() => {
+      const domainsSCAPI:{ [ID:string] : boolean } = {};
+      if (domainsData?.domains) {
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            const doaminSc = domain?.search_console ? JSON.parse(domain.search_console) : {};
+            domainsSCAPI[domain.ID] = doaminSc.client_email && doaminSc.private_key;
+         });
+      }
+      return domainsSCAPI;
+   }, [domainsData]);
+
+   useEffect(() => {
+      if (domainsData?.domains && domainsData.domains.length > 0 && appSettings.screenshot_key) {
+         domainsData.domains.forEach(async (domain:DomainType) => {
+            if (domain.domain) {
+               const domainThumb = await fetchDomainScreenshot(domain.domain, appSettings.screenshot_key || '');
+               if (domainThumb) {
+                  setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domainThumb }));
+               }
+            }
+         });
+      }
+   }, [domainsData, appSettings.screenshot_key]);
+
+   const manuallyUpdateThumb = async (domain: string) => {
+      if (domain && appSettings.screenshot_key) {
+         const domainThumb = await fetchDomainScreenshot(domain, appSettings.screenshot_key, true);
+         if (domainThumb) {
+            toast(`${domain} Screenshot Updated Successfully!`, { icon: '✔️' });
+            setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain]: domainThumb }));
+         } else {
+            toast(`Failed to Fetch ${domain} Screenshot!`, { icon: '⚠️' });
+         }
+      }
+   };
+
+   return (
+      <div data-testid="domains" className="Domain flex flex-col min-h-screen">
+         {((!scraper_type || (scraper_type === 'none')) && !isAppSettingsLoading) && (
+               <div className=' p-3 bg-red-600 text-white text-sm text-center'>
+                  A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app.
+               </div>
+         )}
+         {migrationStatus?.hasMigrations && (
+               <div className=' p-3 bg-black text-white text-sm text-center'>
+                  You need to Update your database. Stop Serpbear and run this command to update your database:
+                  <code className=' bg-gray-700 px-2 py-0 ml-1'>npm run db:migrate</code>
+               </div>
+         )}
+         <Head>
+            <title>Domains - SerpBear</title>
+         </Head>
+         <TopBar showSettings={() => setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} />
+
+         <div className="flex flex-col w-full max-w-5xl mx-auto p-6 lg:mt-24 lg:p-0">
+            <div className='flex justify-between mb-2 items-center'>
+               <div className=' text-sm text-gray-600'>
+                  {domainsData?.domains?.length || 0} Domains <span className=' text-gray-300 ml-1 mr-1'>|</span> {totalKeywords} keywords
+               </div>
+               <div>
+                  <button
+                  data-testid="addDomainButton"
+                  className={'ml-2 inline-block py-2 text-blue-700 font-bold text-sm'}
+                  onClick={() => setShowAddDomain(true)}>
+                     <span
+                     className='text-center leading-4 mr-2 inline-block rounded-full w-7 h-7 pt-1 bg-blue-700 text-white font-bold text-lg'>+</span>
+                     <i className=' not-italic hidden lg:inline-block'>Add Domain</i>
+                  </button>
+               </div>
+            </div>
+            <div className='flex w-full flex-col mb-8'>
+               {domainsData?.domains && domainsData.domains.map((domain:DomainType) => {
+                  return <DomainItem
+                           key={domain.ID}
+                           domain={domain}
+                           selected={false}
+                           isConsoleIntegrated={!!(appSettings && appSettings.search_console_integrated) || !!domainSCAPiObj[domain.ID] }
+                           thumb={domainThumbs[domain.domain]}
+                           updateThumb={manuallyUpdateThumb}
+                           // isConsoleIntegrated={false}
+                           />;
+               })}
+               {isLoading && (
+                  <div className='noDomains mt-4 p-5 py-12 rounded border text-center bg-white text-sm'>
+                     <Icon type="loading" /> Loading Domains...
+                  </div>
+               )}
+               {!isLoading && domainsData && domainsData.domains && domainsData.domains.length === 0 && (
+                  <div className='noDomains mt-4 p-5 py-12 rounded border text-center bg-white text-sm'>
+                     No Domains Found. Add a Domain to get started!
+                  </div>
+               )}
+            </div>
+         </div>
+
+         <CSSTransition in={showAddDomain} timeout={300} classNames="modal_anim" unmountOnExit mountOnEnter>
+            <AddDomain closeModal={() => setShowAddDomain(false)} domains={domainsData?.domains || []} />
+         </CSSTransition>
+         <CSSTransition in={showSettings} timeout={300} classNames="settings_anim" unmountOnExit mountOnEnter>
+             <Settings closeSettings={() => setShowSettings(false)} />
+         </CSSTransition>
+         <Footer currentVersion={appSettings?.version ? appSettings.version : ''} />
+         <Toaster position='bottom-center' containerClassName="react_toaster" />
+      </div>
+   );
+};
+
+export default Domains;