feat: Adds keyword search volume data feature for tracked keywords.

- Adds a volume field in the keyword table.
- Adds a button in the Adwords Integration screen to update all the tracked keywords.
- When a new keyword is added, the volume data is automatically fetched.
- Adds ability to sort keywords based on search volume.
This commit is contained in:
towfiqi 2024-03-01 10:52:45 +06:00
parent 4d15989b28
commit 2a1fc0e43d
16 changed files with 341 additions and 21 deletions

View File

@ -22,6 +22,7 @@ export const dummyKeywords = [
lastUpdated: '2022-11-15T10:49:53.113',
added: '2022-11-11T10:01:06.951',
position: 19,
volume: 10000,
history: {
'2022-11-11': 21,
'2022-11-12': 24,
@ -45,6 +46,7 @@ export const dummyKeywords = [
lastUpdated: '2022-11-15T10:49:53.119',
added: '2022-11-15T10:01:06.951',
position: 29,
volume: 1200,
history: {
'2022-11-11': 33,
'2022-11-12': 34,

View File

@ -36,7 +36,7 @@ const ChartSlim = ({ labels, sreies, noMaxLimit = false }:ChartProps) => {
},
};
return <div className='w-[100px] h-[30px] rounded border border-gray-200'>
return <div className='w-[80px] h-[30px] rounded border border-gray-200'>
<Line
datasetIdKey='XXX'
options={options}

View File

@ -6,6 +6,7 @@ import countries from '../../utils/countries';
import ChartSlim from '../common/ChartSlim';
import KeywordPosition from './KeywordPosition';
import { generateTheChartData } from '../../utils/client/generateChartData';
import { formattedNum } from '../../utils/client/helpers';
type KeywordProps = {
keywordData: KeywordType,
@ -40,7 +41,7 @@ const Keyword = (props: KeywordProps) => {
scDataType = 'threeDays',
} = props;
const {
keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false, volume,
} = keywordData;
const [showOptions, setShowOptions] = useState(false);
const [showPositionError, setPositionError] = useState(false);
@ -99,7 +100,7 @@ const Keyword = (props: KeywordProps) => {
<Icon type="check" size={10} />
</button>
<a
className='py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full lg:max-w-[200px]'
className={`py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full ${showSCData ? 'lg:max-w-[180px]' : 'lg:max-w-[240px]'}`}
onClick={() => showKeywordDetails()}>
<span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />
<span className=' text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-30px)]'>{keyword}{city ? ` (${city})` : ''}</span>
@ -131,12 +132,18 @@ const Keyword = (props: KeywordProps) => {
{chartData.labels.length > 0 && (
<div
className='hidden basis-32 grow-0 cursor-pointer lg:block'
className='hidden basis-20 grow-0 cursor-pointer lg:block'
onClick={() => showKeywordDetails()}>
<ChartSlim labels={chartData.labels} sreies={chartData.sreies} />
</div>
)}
<div
className={`keyword_best hidden bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative lg:block
lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center`}>
{formattedNum(volume)}
</div>
<div
className={`keyword_url inline-block mt-4 mr-5 ml-5 lg:flex-1 text-gray-400 lg:m-0 max-w-[70px]
overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5`}>

View File

@ -76,6 +76,8 @@ const KeywordFilters = (props: KeywordFilterProps) => {
{ value: 'date_desc', label: 'Oldest' },
{ value: 'alpha_asc', label: 'Alphabetically(A-Z)' },
{ value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
{ value: 'vol_asc', label: 'Lowest Search Volume' },
{ value: 'vol_desc', label: 'Highest Search Volume' },
];
if (integratedConsole) {
sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
@ -170,8 +172,8 @@ const KeywordFilters = (props: KeywordFilterProps) => {
{sortOptions && (
<ul
data-testid="sort_options"
className='sort_options mt-2 border absolute min-w-[0] right-0 rounded-lg
max-h-96 bg-white z-[9999] w-44 overflow-y-auto styled-scrollbar'>
className='sort_options mt-2 border absolute w-48 min-w-[0] right-0 rounded-lg
max-h-96 bg-white z-[9999] overflow-y-auto styled-scrollbar'>
{sortOptionChoices.map((sortOption) => {
return <li
key={sortOption.value}

View File

@ -158,9 +158,10 @@ const KeywordsTable = (props: KeywordsTableProps) => {
</span>
<span className='domKeywords_head_position flex-1 basis-24 grow-0 text-center'>Position</span>
<span className='domKeywords_head_best flex-1 basis-16 grow-0 text-center'>Best</span>
<span className='domKeywords_head_history flex-1 basis-32 grow-0 '>History (7d)</span>
<span className='domKeywords_head_history flex-1 basis-20 grow-0'>History (7d)</span>
<span className='domKeywords_head_volume flex-1 basis-24 grow-0 text-center'>Volume</span>
<span className='domKeywords_head_url flex-1'>URL</span>
<span className='domKeywords_head_updated flex-1'>Updated</span>
<span className='domKeywords_head_updated flex-1 relative left-3'>Updated</span>
{showSCData && (
<div className='domKeywords_head_sc flex-1 min-w-[170px] mr-7 text-center'>
{/* Search Console */}

View File

@ -1,5 +1,5 @@
import React from 'react';
import { useTestAdwordsIntegration } from '../../services/adwords';
import { useMutateKeywordsVolume, useTestAdwordsIntegration } from '../../services/adwords';
import Icon from '../common/Icon';
import SecretField from '../common/SecretField';
@ -24,7 +24,10 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
} = settings || {};
const { mutate: testAdWordsIntegration, isLoading: isTesting } = useTestAdwordsIntegration();
const { mutate: getAllVolumeData, isLoading: isUpdatingVolume } = useMutateKeywordsVolume();
const cloudProjectIntegrated = adwords_client_id && adwords_client_secret && adwords_refresh_token;
const hasAllCredentials = adwords_client_id && adwords_client_secret && adwords_refresh_token && adwords_developer_token && adwords_account_id;
const udpateAndAuthenticate = () => {
if (adwords_client_id && adwords_client_secret) {
@ -40,11 +43,17 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
};
const testIntegration = () => {
if (adwords_client_id && adwords_client_secret && adwords_refresh_token && adwords_developer_token && adwords_account_id) {
if (hasAllCredentials) {
testAdWordsIntegration({ developer_token: adwords_developer_token, account_id: adwords_account_id });
}
};
const updateVolumeData = () => {
if (hasAllCredentials) {
getAllVolumeData({ domain: 'all' });
}
};
return (
<div>
<div>
@ -98,9 +107,9 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
<div className="settings__section__input mb-4 flex justify-between items-center w-full">
<button
className={`py-2 px-5 w-full text-sm font-semibold rounded bg-indigo-50 text-blue-700 border border-indigo-100
${adwords_client_id && adwords_client_secret && adwords_refresh_token ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
${hasAllCredentials ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
hover:bg-blue-700 hover:text-white transition`}
title='Insert All the data in the above fields to Authenticate'
title={hasAllCredentials ? '' : 'Insert All the data in the above fields to Test the Integration'}
onClick={testIntegration}>
{isTesting && <Icon type='loading' />}
<Icon type='adwords' size={14} /> Test AdWords Integration
@ -108,6 +117,24 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
</div>
</div>
</div>
<div className='mt-4 mb-4 border-b border-gray-100 pt-4 pb-0 relative'>
{!hasAllCredentials && <div className=' absolute w-full h-full z-50' />}
<h4 className=' mb-3 font-semibold text-blue-700'>Update Keyword Volume Data</h4>
<div className={!hasAllCredentials ? 'opacity-40' : ''}>
<div className="settings__section__input mb-4 flex justify-between items-center w-full">
<p>Update Volume data for all your Tracked Keywords.</p>
</div>
<div className="settings__section__input mb-4 flex justify-between items-center w-full">
<button
className={`py-2 px-5 w-full text-sm font-semibold rounded bg-indigo-50 text-blue-700 border border-indigo-100
${hasAllCredentials ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
hover:bg-blue-700 hover:text-white transition`}
onClick={updateVolumeData}>
<Icon type={isUpdatingVolume ? 'loading' : 'reload'} size={isUpdatingVolume ? 16 : 12} /> Update Keywords Volume
</button>
</div>
</div>
</div>
<p className='mb-4 text-xs'>
<a target='_blank' rel='noreferrer' href='https://docs.serpbear.com/keyword-research' className=' underline text-blue-600'>Integrate Google Adwords</a> to get Keyword Ideas & Search Volume.{' '}
</p>

View File

@ -0,0 +1,35 @@
// Migration: Adds volume field to the keyword table.
// CLI Migration
module.exports = {
up: async (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(async (t) => {
try {
const keywordTableDefinition = await queryInterface.describeTable('keyword');
if (keywordTableDefinition) {
if (!keywordTableDefinition.volume) {
await queryInterface.addColumn('keyword', 'volume', {
type: Sequelize.DataTypes.STRING, allowNull: false, defaultValue: 0,
}, { transaction: t });
}
}
} catch (error) {
console.log('error :', error);
}
});
},
down: (queryInterface) => {
return queryInterface.sequelize.transaction(async (t) => {
try {
const keywordTableDefinition = await queryInterface.describeTable('keyword');
if (keywordTableDefinition) {
if (keywordTableDefinition.volume) {
await queryInterface.removeColumn('keyword', 'volume', { transaction: t });
}
}
} catch (error) {
console.log('error :', error);
}
});
},
};

View File

@ -47,6 +47,9 @@ class Keyword extends Model {
@Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) })
history!: string;
@Column({ type: DataType.INTEGER, allowNull: false, defaultValue: 0 })
volume!: number;
@Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) })
url!: string;

View File

@ -7,6 +7,7 @@ import verifyUser from '../../utils/verifyUser';
import parseKeywords from '../../utils/parseKeywords';
import { integrateKeywordSCData, readLocalSCData } from '../../utils/searchConsole';
import refreshAndUpdateKeywords from '../../utils/refresh';
import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords';
type KeywordsGetResponse = {
keywords?: KeywordType[],
@ -103,8 +104,20 @@ const addKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
const newKeywords:Keyword[] = await Keyword.bulkCreate(keywordsToAdd);
const formattedkeywords = newKeywords.map((el) => el.get({ plain: true }));
const keywordsParsed: KeywordType[] = parseKeywords(formattedkeywords);
// Queue the SERP Scraping Process
const settings = await getAppSettings();
refreshAndUpdateKeywords(newKeywords, settings); // Queue the SERP Scraping Process
refreshAndUpdateKeywords(newKeywords, settings);
// Update the Keyword Volume
const { adwords_account_id, adwords_client_id, adwords_client_secret, adwords_developer_token } = settings;
if (adwords_account_id && adwords_client_id && adwords_client_secret && adwords_developer_token) {
const keywordsVolumeData = await getKeywordsVolume(keywordsParsed);
if (keywordsVolumeData.volumes !== false) {
await updateKeywordsVolumeData(keywordsVolumeData.volumes);
}
}
return res.status(201).json({ keywords: keywordsParsed });
} catch (error) {
console.log('[ERROR] Adding New Keywords ', error);

View File

@ -93,6 +93,7 @@ const getKeywordSearchResults = async (req: NextApiRequest, res: NextApiResponse
country: req.query.country as string,
domain: '',
lastUpdated: '',
volume: 0,
added: '',
position: 111,
sticky: false,

67
pages/api/volume.ts Normal file
View File

@ -0,0 +1,67 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { Op } from 'sequelize';
import db from '../../database/database';
import Keyword from '../../database/models/keyword';
import verifyUser from '../../utils/verifyUser';
import parseKeywords from '../../utils/parseKeywords';
import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords';
type KeywordsRefreshRes = {
keywords?: KeywordType[]
error?: string|null,
}
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
await db.sync();
const authorized = verifyUser(req, res);
if (authorized !== 'authorized') {
return res.status(401).json({ error: authorized });
}
if (req.method === 'POST') {
return updatekeywordVolume(req, res);
}
return res.status(502).json({ error: 'Unrecognized Route.' });
}
const updatekeywordVolume = async (req: NextApiRequest, res: NextApiResponse<KeywordsRefreshRes>) => {
const { keywords = [], domain = '', update = true } = req.body || {};
if (keywords.length === 0 && !domain) {
return res.status(400).json({ error: 'Please provide keyword Ids or a domain name.' });
}
try {
let keywordsToSend: KeywordType[] = [];
if (keywords.length > 0) {
const allKeywords:Keyword[] = await Keyword.findAll({ where: { ID: { [Op.in]: keywords } } });
keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
}
if (domain) {
// const allDomain = domain === 'all';
// const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
// keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
}
if (keywordsToSend.length > 0) {
const keywordsVolumeData = await getKeywordsVolume(keywordsToSend);
// console.log('keywordsVolumeData :', keywordsVolumeData);
if (keywordsVolumeData.error) {
return res.status(400).json({ keywords: [], error: keywordsVolumeData.error });
}
if (keywordsVolumeData.volumes !== false) {
if (update) {
const updated = await updateKeywordsVolumeData(keywordsVolumeData.volumes);
if (updated) {
return res.status(200).json({ keywords });
}
}
} else {
return res.status(400).json({ error: 'Error Fetching Keywords Volume Data from Google Adwords' });
}
}
return res.status(400).json({ keywords: [], error: 'Error Updating Keywords Volume data' });
} catch (error) {
console.log('[Error] updating keywords Volume Data: ', error);
return res.status(400).json({ error: 'Error Updating Keywords Volume data' });
}
};

View File

@ -99,3 +99,30 @@ export function useMutateFavKeywordIdeas(router:NextRouter, onSuccess?: Function
},
});
}
export function useMutateKeywordsVolume(onSuccess?: Function) {
return useMutation(async (data:Record<string, any>) => {
const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' });
const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ ...data }) };
const res = await fetch(`${window.location.origin}/api/volume`, fetchOpts);
if (res.status >= 400 && res.status < 600) {
const errorData = await res.json();
throw new Error(errorData?.error ? errorData.error : 'Bad response from server');
}
return res.json();
}, {
onSuccess: async (data) => {
toast('Keyword Volume Data Loaded Successfully! Reloading Page...', { icon: '✔️' });
if (onSuccess) {
onSuccess(false);
}
setTimeout(() => {
window.location.reload();
}, 3000);
},
onError: (error) => {
console.log('Error Loading Keyword Volume Data!!!', error);
toast('Error Loading Keyword Volume Data', { icon: '⚠️' });
},
});
}

View File

@ -16,6 +16,7 @@ module.exports = {
'w-[240px]',
'min-w-[270px]',
'min-w-[180px]',
'max-w-[180px]',
],
theme: {
extend: {},

1
types.d.ts vendored
View File

@ -32,6 +32,7 @@ type KeywordType = {
lastUpdated: string,
added: string,
position: number,
volume: number,
sticky: boolean,
history: KeywordHistory,
lastResult: KeywordLastResult[],

View File

@ -1,6 +1,7 @@
import { readFile, writeFile } from 'fs/promises';
import Cryptr from 'cryptr';
import TTLCache from '@isaacs/ttlcache';
import { setTimeout as sleep } from 'timers/promises';
import Keyword from '../database/models/keyword';
import parseKeywords from './parseKeywords';
import countries from './countries';
@ -8,15 +9,17 @@ import { readLocalSCData } from './searchConsole';
const memoryCache = new TTLCache({ max: 10000 });
type keywordIdeasMetrics = {
competition: IdeaKeyword['competition'],
monthlySearchVolumes: {month : string, year : string, monthlySearches : string}[],
avgMonthlySearches: string,
competitionIndex: string,
lowTopOfPageBidMicros: string,
highTopOfPageBidMicros: string
}
type keywordIdeasResponseItem = {
keywordIdeaMetrics: {
competition: IdeaKeyword['competition'],
monthlySearchVolumes: {month : string, year : string, monthlySearches : string}[],
avgMonthlySearches: string,
competitionIndex: string,
lowTopOfPageBidMicros: string,
highTopOfPageBidMicros: string
},
keywordIdeaMetrics: keywordIdeasMetrics,
text: string,
keywordAnnotations: Object
};
@ -250,6 +253,130 @@ const extractAdwordskeywordIdeas = (keywordIdeas:keywordIdeasResponseItem[], opt
return keywords.sort((a: IdeaKeyword, b: IdeaKeyword) => (b.avgMonthlySearches > a.avgMonthlySearches ? 1 : -1));
};
/**
* Retrieves keyword search volumes from Google Adwords API based on provided keywords and their countries.
* @param {KeywordType[]} keywords - The keywords that you want to get the search volume data for.
* @returns returns a Promise that resolves to an object with a `volumes` and error `proprties`.
* The `volumes` propery which outputs `false` if the request fails and outputs the volume data in `{[keywordID]: volume}` object if succeeds.
* The `error` porperty that outputs the error message if any.
*/
export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error?: string, volumes: false | Record<number, number>}> => {
const credentials = await getAdwordsCredentials();
if (!credentials) { return { error: 'Cannot Load Adwords Credentials', volumes: false }; }
const { client_id, client_secret, developer_token, account_id } = credentials;
if (!client_id || !client_secret || !developer_token || !account_id) {
return { error: 'Adwords Not Integrated Properly', volumes: false };
}
// Generate Access Token
let accessToken = '';
const cachedAccessToken:string|false|undefined = memoryCache.get('adwords_token');
if (cachedAccessToken && !test) {
accessToken = cachedAccessToken;
} else {
accessToken = await getAdwordsAccessToken(credentials);
memoryCache.delete('adwords_token');
memoryCache.set('adwords_token', accessToken, { ttl: 3300000 });
}
const fetchedKeywords:Record<number, number> = {};
if (accessToken) {
// Group keywords based on their country.
const keywordRequests: Record<string, KeywordType[]> = {};
keywords.forEach((kw) => {
const kwCountry = kw.country;
if (keywordRequests[kwCountry]) {
keywordRequests[kwCountry].push(kw);
} else {
keywordRequests[kwCountry] = [kw];
}
});
// Send Requests to adwords based on grouped countries.
// Since adwords does not allow sending country data for each keyword we are making requests for.
for (const country in keywordRequests) {
if (Object.hasOwn(keywordRequests, country) && keywordRequests[country].length > 0) {
try {
// API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordHistoricalMetrics
const customerID = account_id.replaceAll('-', '');
const geoTargetConstants = countries[country][3]; // '2840';
const reqKeywords = keywordRequests[country].map((kw) => kw.keyword);
const reqPayload: Record<string, any> = {
keywords: [...new Set(reqKeywords)],
geoTargetConstants: `geoTargetConstants/${geoTargetConstants}`,
// language: `languageConstants/${language}`,
};
const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordHistoricalMetrics`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'developer-token': developer_token,
Authorization: `Bearer ${accessToken}`,
'login-customer-id': customerID,
},
body: JSON.stringify(reqPayload),
});
const ideaData = await resp.json();
if (resp.status !== 200) {
console.log('[ERROR] Adwords Volume Request Response :', ideaData?.error?.details[0]?.errors[0]?.message);
console.log('Response from AdWords :', JSON.stringify(ideaData, null, 2));
}
if (ideaData?.results) {
if (Array.isArray(ideaData.results) && ideaData.results.length > 0) {
const volumeDataObj:Map<string, number> = new Map();
ideaData.results.forEach((item:{ keywordMetrics: keywordIdeasMetrics, text: string }) => {
const kwVol = item?.keywordMetrics?.avgMonthlySearches;
volumeDataObj.set(`${country}:${item.text}`, kwVol ? parseInt(kwVol, 10) : 0);
});
keywordRequests[country].forEach((keyword) => {
const keywordKey = `${keyword.country}:${keyword.keyword}`;
if (volumeDataObj.has(keywordKey)) {
const volume = volumeDataObj.get(keywordKey);
if (volume !== undefined) {
fetchedKeywords[keyword.ID] = volume;
}
}
});
// console.log('fetchedKeywords :', fetchedKeywords);
}
}
} catch (error) {
console.log('[ERROR] Fetching Keyword Volume from Adwords :', error);
}
if (Object.keys(keywordRequests).length > 1) {
await sleep(7000);
}
}
}
}
return { volumes: fetchedKeywords };
};
/**
* Updates volume data for keywords in the Keywords database using async/await and error handling.
* @param {false | Record<number, number>} volumesData - The `volumesData` parameter can either be `false` or an object containing
* keyword IDs as keys and corresponding volume data as values.
* @returns returns a Promise that resolves to `true` if `volumesData` is not `false` else it returns `false`.
*/
export const updateKeywordsVolumeData = async (volumesData: false | Record<number, number>) => {
if (volumesData === false) { return false; }
Object.keys(volumesData).forEach(async (keywordID) => {
const keyID = parseInt(keywordID, 10);
const volumeData = volumesData && volumesData[keyID] ? volumesData[keyID] : 0;
try {
await Keyword.update({ volume: volumeData }, { where: { ID: keyID } });
} catch (error) {
console.log('');
}
});
return true;
};
/**
* The function `getLocalKeywordIdeas` reads keyword ideas data from a local JSON file based on a domain slug and returns it as a Promise.
* @param {string} domain - The `domain` parameter is the domain slug for which the keyword Ideas are fetched.

View File

@ -28,6 +28,12 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp
case 'alpha_desc':
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.keyword > b.keyword ? 1 : -1));
break;
case 'vol_asc':
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (b.volume - a.volume));
break;
case 'vol_desc':
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.volume - b.volume));
break;
case 'imp_desc':
if (scDataType) {
sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {