UniqueSoft/serpbear
2
0
Fork 0
mirror of https://github.com/towfiqi/serpbear synced 2025-06-26 18:15:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Adds the Ability to set Search Console Property type via Domain Settings.

Browse Source 
- Previously only domain properties worked with SerpBear. This feature adds the ability to add URL properties as well.
- Adds a new field "search_console" in Domain Table.
- Adds a new Search Console option in Domain Settings Modal UI.
- When the  new "This is a URL Property" option is enabled, the exact Property URL should be provided.

closes #50
This commit is contained in:
towfiqi 2024-02-06 23:42:28 +06:00
parent 1041cb3c0b
commit b2e97b2ebe
12 changed files with 182 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintrc.json
View File

@ -13,6 +13,7 @@
"arrow-body-style":"off",
"max-len": ["error", {"code": 150, "ignoreComments": true, "ignoreUrls": true}],
"import/no-extraneous-dependencies": "off",
"no-unused-vars": "off",
"import/extensions": [
"error",
"ignorePackages",

28
components/common/InputField.tsx Normal file
View File

@ -0,0 +1,28 @@
type InputFieldProps = {
label: string;
value: string;
onChange: Function;
placeholder?: string;
classNames?: string;
hasError?: boolean;
}
const InputField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: InputFieldProps) => {
const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize';
return (
<div className="field--input w-full relative">
<label className={labelStyle}>{label}</label>
<input
className={`w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none
focus:border-blue-200 ${hasError ? ' border-red-400 focus:border-red-400' : ''} `}
type={'text'}
value={value}
onChange={(event) => onChange(event.target.value)}
autoComplete="off"
placeholder={placeholder}
/>
</div>
);
};
export default InputField;

32
components/common/ToggleField.tsx Normal file
View File

@ -0,0 +1,32 @@
type ToggleFieldProps = {
label: string;
value: string;
onChange: (bool:boolean) => void ;
classNames?: string;
}
const ToggleField = ({ label = '', value = '', onChange, classNames = '' }: ToggleFieldProps) => {
return (
<div className={`field--toggle w-full relative ${classNames}`}>
<label className="relative inline-flex items-center cursor-pointer w-full justify-between">
<span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-56">{label}</span>
<input
type="checkbox"
value={value}
checked={!!value}
className="sr-only peer"
onChange={() => onChange(!value)}
/>
<div className="relative rounded-3xl w-9 h-5 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300
after:border after:rounded-full after:h-4 after:w-4
after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
</label>
</div>
);
};
export default ToggleField;

81
components/domains/DomainSettings.tsx
View File

@ -3,6 +3,8 @@ import { useState } from 'react';
import Icon from '../common/Icon';
import Modal from '../common/Modal';
import { useDeleteDomain, useUpdateDomain } from '../../services/domains';
import InputField from '../common/InputField';
import SelectField from '../common/SelectField';
type DomainSettingsProps = {
domain:DomainType|false,
@ -16,11 +18,13 @@ type DomainSettingsError = {
const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
const router = useRouter();
const [currentTab, setCurrentTab] = useState<'notification'|'searchconsole'>('notification');
const [showRemoveDomain, setShowRemoveDomain] = useState<boolean>(false);
const [settingsError, setSettingsError] = useState<DomainSettingsError>({ type: '', msg: '' });
const [domainSettings, setDomainSettings] = useState<DomainSettings>(() => ({
notification_interval: domain && domain.notification_interval ? domain.notification_interval : 'never',
notification_emails: domain && domain.notification_emails ? domain.notification_emails : '',
search_console: domain && domain.search_console ? JSON.parse(domain.search_console) : { property_type: 'domain', url: '' },
}));
const { mutate: updateMutate } = useUpdateDomain(() => closeModal(false));
@ -29,10 +33,6 @@ const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
router.push('/domains');
});
const updateNotiEmails = (event:React.FormEvent<HTMLInputElement>) => {
setDomainSettings({ ...domainSettings, notification_emails: event.currentTarget.value });
};
const updateDomain = () => {
console.log('Domain: ');
let error: DomainSettingsError | null = null;
@ -55,24 +55,73 @@ const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => {
}
};
const tabStyle = 'inline-block px-4 py-1 rounded-full mr-3 cursor-pointer text-sm select-none';
return (
<div>
<Modal closeModal={() => closeModal(false)} title={'Domain Settings'} width="[500px]">
<div data-testid="domain_settings" className=" text-sm">
<div className="mb-6 flex justify-between items-center">
<h4>Notification Emails
{settingsError.type === 'email' && <span className="text-red-500 font-semibold ml-2">{settingsError.msg}</span>}
</h4>
<input
className={`border w-46 text-sm transition-all rounded p-1.5 px-4 outline-none ring-0
${settingsError.type === 'email' ? ' border-red-300' : ''}`}
type="text"
placeholder='Your Emails'
onChange={updateNotiEmails}
value={domainSettings.notification_emails || ''}
/>
<div className='mt-5 mb-5 '>
<ul>
<li
className={`${tabStyle} ${currentTab === 'notification' ? ' bg-blue-50 text-blue-600' : ''}`}
onClick={() => setCurrentTab('notification')}>
Notification
</li>
<li
className={`${tabStyle} ${currentTab === 'searchconsole' ? ' bg-blue-50 text-blue-600' : ''}`}
onClick={() => setCurrentTab('searchconsole')}>
Search Console
</li>
</ul>
</div>
<div>
{currentTab === 'notification' && (
<div className="mb-4 flex justify-between items-center w-full">
<InputField
label='Notification Emails'
onChange={(emails:string) => setDomainSettings({ ...domainSettings, notification_emails: emails })}
value={domainSettings.notification_emails || ''}
placeholder='Your Emails'
/>
</div>
)}
{currentTab === 'searchconsole' && (
<>
<div className="mb-4 flex justify-between items-center w-full">
<label className='mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'>Property Type</label>
<SelectField
options={[{ label: 'Domain', value: 'domain' }, { label: 'URL', value: 'url' }]}
selected={[domainSettings.search_console?.property_type || 'domain']}
defaultLabel="Select Search Console Property Type"
updateField={(updated:['domain'|'url']) => setDomainSettings({
...domainSettings,
search_console: { ...domainSettings.search_console, property_type: updated[0] || 'domain' },
})}
multiple={false}
rounded={'rounded'}
/>
</div>
{domainSettings?.search_console?.property_type === 'url' && (
<div className="mb-4 flex justify-between items-center w-full">
<InputField
label='Property URL (Required)'
onChange={(url:string) => setDomainSettings({
...domainSettings,
search_console: { ...domainSettings.search_console, url },
})}
value={domainSettings?.search_console?.url || ''}
placeholder='Search Console Property URL. eg: https://mywebsite.com/'
/>
</div>
)}
</>
)}
</div>
</div>
<div className="flex justify-between border-t-[1px] border-gray-100 mt-8 pt-4 pb-0">
<button
className="text-sm font-semibold text-red-500"

23
components/settings/ScraperSettings.tsx
View File

@ -3,6 +3,7 @@ import { useClearFailedQueue } from '../../services/settings';
import Icon from '../common/Icon';
import SelectField, { SelectionOption } from '../common/SelectField';
import SecretField from '../common/SecretField';
import ToggleField from '../common/ToggleField';
type ScraperSettingsProps = {
settings: SettingsType,
@ -108,23 +109,11 @@ const ScraperSettings = ({ settings, settingsError, updateSettings }:ScraperSett
<small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
</div>
<div className="settings__section__input mb-5">
<label className="relative inline-flex items-center cursor-pointer w-full justify-between">
<span className="text-sm font-medium text-gray-900 dark:text-gray-300 w-56">Auto Retry Failed Keyword Scrape</span>
<input
type="checkbox"
value={settings?.scrape_retry ? 'true' : '' }
checked={settings.scrape_retry || false}
className="sr-only peer"
onChange={() => updateSettings('scrape_retry', !settings.scrape_retry)}
/>
<div className="relative rounded-3xl w-9 h-5 bg-gray-200 peer-focus:outline-none peer-focus:ring-4
peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800rounded-full peer dark:bg-gray-700
peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300
after:border after:rounded-full after:h-4 after:w-4
after:transition-all dark:border-gray-600 peer-checked:bg-blue-600"></div>
</label>
<ToggleField
label='Auto Retry Failed Keyword Scrape'
value={settings?.scrape_retry ? 'true' : '' }
onChange={(val) => updateSettings('scrape_retry', val)}
/>
</div>
{settings?.scrape_retry && (settings.failed_queue?.length || 0) > 0 && (
<div className="settings__section__input mb-5">

15
database/migrations/1707233039698-add-domain-searchconsole-field.js Normal file
View File

@ -0,0 +1,15 @@
// Migration: Adds search_console field to domain table to assign search console property type, url and api.
// CLI Migration
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(async (t) => {
await queryInterface.addColumn('domain', 'search_console', { type: Sequelize.DataTypes.STRING }, { transaction: t });
});
},
down: (queryInterface) => {
return queryInterface.sequelize.transaction(async (t) => {
await queryInterface.removeColumn('domain', 'search_console', { transaction: t });
});
},
};

3
database/models/domain.ts
View File

@ -38,6 +38,9 @@ class Domain extends Model {
@Column({ type: DataType.STRING, allowNull: true, defaultValue: '' })
notification_emails!: string;
@Column({ type: DataType.STRING, allowNull: true })
search_console!: string;
}
export default Domain;

4
pages/api/domains.ts
View File

@ -108,12 +108,12 @@ export const updateDomain = async (req: NextApiRequest, res: NextApiResponse<Dom
return res.status(400).json({ domain: null, error: 'Domain is Required!' });
}
const { domain } = req.query || {};
const { notification_interval, notification_emails } = req.body;
const { notification_interval, notification_emails, search_console } = req.body as DomainSettings;
try {
const domainToUpdate: Domain|null = await Domain.findOne({ where: { domain } });
if (domainToUpdate) {
domainToUpdate.set({ notification_interval, notification_emails });
domainToUpdate.set({ notification_interval, notification_emails, search_console: JSON.stringify(search_console) });
await domainToUpdate.save();
}
return res.status(200).json({ domain: domainToUpdate });

7
pages/api/insight.ts
View File

@ -3,6 +3,7 @@ import db from '../../database/database';
import { getCountryInsight, getKeywordsInsight, getPagesInsight } from '../../utils/insight';
import { fetchDomainSCData, readLocalSCData } from '../../utils/searchConsole';
import verifyUser from '../../utils/verifyUser';
import Domain from '../../database/models/domain';
type SCInsightRes = {
data: InsightDataType | null,
@ -49,7 +50,11 @@ const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiRe
// If the Local SC Domain Data file does not exist, fetch from Googel Search Console.
try {
const scData = await fetchDomainSCData(domainname);
const query = { domain: domainname };
const foundDomain:Domain| null = await Domain.findOne({ where: query });
const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
const scData = await fetchDomainSCData(domainObj);
const response = getInsightFromSCData(scData);
return res.status(200).json({ data: response });
} catch (error) {

9
pages/api/searchconsole.ts
View File

@ -42,7 +42,10 @@ const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiRespo
return res.status(200).json({ data: localSCData });
}
try {
const scData = await fetchDomainSCData(domainname);
const query = { domain: domainname };
const foundDomain:Domain| null = await Domain.findOne({ where: query });
const domainObj: DomainType = foundDomain && foundDomain.get({ plain: true });
const scData = await fetchDomainSCData(domainObj);
return res.status(200).json({ data: scData });
} catch (error) {
console.log('[ERROR] Getting Search Console Data for: ', domainname, error);
@ -53,9 +56,9 @@ const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiRespo
const cronRefreshSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse<searchConsoleCRONRes>) => {
try {
const allDomainsRaw = await Domain.findAll();
const Domains: Domain[] = allDomainsRaw.map((el) => el.get({ plain: true }));
const Domains: DomainType[] = allDomainsRaw.map((el) => el.get({ plain: true }));
for (const domain of Domains) {
await fetchDomainSCData(domain.domain);
await fetchDomainSCData(domain);
}
return res.status(200).json({ status: 'completed' });
} catch (error) {

7
types.d.ts vendored
View File

@ -15,6 +15,7 @@ type DomainType = {
scVisits?: number,
scImpressions?: number,
scPosition?: number,
search_console?: string,
}
type KeywordHistory = {
@ -62,9 +63,15 @@ type countryCodeData = {
[ISO:string] : string
}
type DomainSearchConsole = {
property_type?: 'domain' | 'url',
url?: string
}
type DomainSettings = {
notification_interval: string,
notification_emails: string,
search_console?: DomainSearchConsole
}
type SettingsType = {

19
utils/searchConsole.ts
View File

@ -11,13 +11,15 @@ type fetchConsoleDataResponse = SearchAnalyticsItem[] | SearchAnalyticsStat[] |
/**
* function that retrieves data from the Google Search Console API based on the provided domain name, number of days, and optional type.
* @param {string} domainName - The domain name for which you want to fetch search console data.
* @param {DomainType} domain - The domain for which you want to fetch search console data.
* @param {number} days - The `days` parameter is the number of days of data you want to fetch from the Search Console.
* @param {string} [type] - The `type` parameter is an optional parameter that specifies the type of data to fetch from the Search Console API.
* @returns {Promise<fetchConsoleDataResponse>}
*/
const fetchSearchConsoleData = async (domainName:string, days:number, type?:string): Promise<fetchConsoleDataResponse> => {
if (!domainName) return { error: true, errorMsg: 'Domain Not Provided!' };
const fetchSearchConsoleData = async (domain:DomainType, days:number, type?:string): Promise<fetchConsoleDataResponse> => {
if (!domain) return { error: true, errorMsg: 'Domain Not Provided!' };
const domainName = domain.domain;
const domainSettings = domain.search_console ? JSON.parse(domain.search_console) : { property_type: 'domain', url: '' };
try {
const authClient = new auth.GoogleAuth({
credentials: {
@ -51,7 +53,8 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
};
}
const res = client.searchanalytics.query({ siteUrl: `sc-domain:${domainName}`, requestBody });
const siteUrl = domainSettings.property_type === 'url' && domainSettings.url ? domainSettings.url : `sc-domain:${domainName}`;
const res = client.searchanalytics.query({ siteUrl, requestBody });
const resData:any = (await res).data;
let finalRows = resData.rows ? resData.rows.map((item:SearchAnalyticsRawItem) => parseSearchConsoleItem(item, domainName)) : [];
@ -79,15 +82,15 @@ const fetchSearchConsoleData = async (domainName:string, days:number, type?:stri
/**
* The function fetches search console data for a given domain and returns it in a structured format.
* @param {string} domain - The `domain` parameter is a string that represents the domain for which we
* @param {DomainType} domain - The `domain` parameter is a Domain object that represents the domain for which we
* want to fetch search console data.
* @returns The function `fetchDomainSCData` is returning a Promise that resolves to an object of type
* `SCDomainDataType`.
*/
export const fetchDomainSCData = async (domain:string): Promise<SCDomainDataType> => {
export const fetchDomainSCData = async (domain:DomainType): Promise<SCDomainDataType> => {
const days = [3, 7, 30];
const scDomainData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '', stats: [] };
if (domain) {
if (domain.domain) {
for (const day of days) {
const items = await fetchSearchConsoleData(domain, day);
scDomainData.lastFetched = new Date().toJSON();
@ -103,7 +106,7 @@ export const fetchDomainSCData = async (domain:string): Promise<SCDomainDataType
if (stats && Array.isArray(stats) && stats.length > 0) {
scDomainData.stats = stats as SearchAnalyticsStat[];
}
await updateLocalSCData(domain, scDomainData);
await updateLocalSCData(domain.domain, scDomainData);
}
return scDomainData;