import { readFile, writeFile } from 'fs/promises'; import Cryptr from 'cryptr'; import TTLCache from '@isaacs/ttlcache'; import { setTimeout as sleep } from 'timers/promises'; import Keyword from '../database/models/keyword'; import parseKeywords from './parseKeywords'; import countries from './countries'; import { readLocalSCData } from './searchConsole'; const memoryCache = new TTLCache({ max: 10000 }); type keywordIdeasMetrics = { competition: IdeaKeyword['competition'], monthlySearchVolumes: {month : string, year : string, monthlySearches : string}[], avgMonthlySearches: string, competitionIndex: string, lowTopOfPageBidMicros: string, highTopOfPageBidMicros: string } type keywordIdeasResponseItem = { keywordIdeaMetrics: keywordIdeasMetrics, text: string, keywordAnnotations: Object }; type IdeaSettings = { country?: string; city?: string; language?: string; keywords?: string[]; url?: string; domain?:string; seedType: 'auto' | 'custom' | 'tracking' | 'searchconsole' } type IdeaDatabaseUpdateData = { keywords?: IdeaKeyword[], settings?: IdeaSettings, favorites?: IdeaKeyword[] } export type KeywordIdeasDatabase = { keywords: IdeaKeyword[], favorites: IdeaKeyword[], settings: IdeaSettings, updated: number } /** * The function `getAdwordsCredentials` reads and decrypts Google Ads credentials from the App settings file. * @returns {Promise} returns either a decrypted `AdwordsCredentials` object if the settings are successfully decrypted, * or `false` if the decryption process fails. */ export const getAdwordsCredentials = async (): Promise => { try { const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' }); const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {}; let decryptedSettings: false | AdwordsCredentials = false; try { const cryptr = new Cryptr(process.env.SECRET as string); const client_id = settings.adwords_client_id ? cryptr.decrypt(settings.adwords_client_id) : ''; const client_secret = settings.adwords_client_secret ? cryptr.decrypt(settings.adwords_client_secret) : ''; const developer_token = settings.adwords_developer_token ? cryptr.decrypt(settings.adwords_developer_token) : ''; const account_id = settings.adwords_account_id ? cryptr.decrypt(settings.adwords_account_id) : ''; const refresh_token = settings.adwords_refresh_token ? cryptr.decrypt(settings.adwords_refresh_token) : ''; decryptedSettings = { client_id, client_secret, developer_token, account_id, refresh_token, }; } catch (error) { console.log('Error Decrypting Settings API Keys!'); } return decryptedSettings; } catch (error) { console.log('[ERROR] Getting App Settings. ', error); } return false; }; /** * retrieves an access token using Google Ads credentials for Google API authentication. * @param {AdwordsCredentials} credentials - The `credentials` to use to generate the access token, * @returns {Promise} the fetched access token or an empty string if failed. */ export const getAdwordsAccessToken = async (credentials:AdwordsCredentials) => { const { client_id, client_secret, refresh_token } = credentials; try { const resp = await fetch('https://www.googleapis.com/oauth2/v3/token', { method: 'POST', body: new URLSearchParams({ grant_type: 'refresh_token', client_id, client_secret, refresh_token }), }); const tokens = await resp.json(); // console.log('token :', tokens); return tokens?.access_token || ''; } catch (error) { console.log('[Error] Getting Google Account Access Token:', error); return ''; } }; /** * The function `getAdwordsKeywordIdeas` retrieves keyword ideas from Google Ads API based on * provided credentials and settings. * @param {AdwordsCredentials} credentials - an object containing Google Ads credentials needed to authenticate * the API request. * @param {IdeaSettings} adwordsDomainOptions - an object that contains settings and options for fetching * keyword ideas from Google Ads. * @param {boolean} [test=false] - a boolean flag that indicates whether the function is being run in a test mode or not. * When `test` is set to `true`, only 1 keyword is requested from adwords. * @returns returns an array of fetched keywords (`fetchedKeywords`) after processing the Google Ads API response. */ export const getAdwordsKeywordIdeas = async (credentials:AdwordsCredentials, adwordsDomainOptions:IdeaSettings, test:boolean = false) => { if (!credentials) { return false; } const { account_id, developer_token } = credentials; const { country = '2840', language = '1000', keywords = [], domain = '', seedType } = adwordsDomainOptions || {}; let accessToken = ''; const cachedAccessToken:string|false|undefined = memoryCache.get('adwords_token'); if (cachedAccessToken && !test) { accessToken = cachedAccessToken; } else { accessToken = await getAdwordsAccessToken(credentials); memoryCache.delete('adwords_token'); memoryCache.set('adwords_token', accessToken, { ttl: 3300000 }); } let fetchedKeywords:IdeaKeyword[] = []; if (accessToken) { const seedKeywords = [...keywords]; // Load Keywords from Google Search Console File. if (seedType === 'searchconsole' && domain) { const domainSCData = await readLocalSCData(domain); if (domainSCData && domainSCData.thirtyDays) { const scKeywords = domainSCData.thirtyDays; const sortedSCKeywords = scKeywords.sort((a, b) => (b.impressions > a.impressions ? 1 : -1)); sortedSCKeywords.slice(0, 100).forEach((sckeywrd) => { if (sckeywrd.keyword && !seedKeywords.includes(sckeywrd.keyword)) { seedKeywords.push(sckeywrd.keyword); } }); } } // Load all Keywords from Database if (seedType === 'tracking' && domain) { const allKeywords:Keyword[] = await Keyword.findAll({ where: { domain } }); const currentKeywords: KeywordType[] = parseKeywords(allKeywords.map((e) => e.get({ plain: true }))); currentKeywords.forEach((keyword) => { if (keyword.keyword && !seedKeywords.includes(keyword.keyword)) { seedKeywords.push(keyword.keyword); } }); } try { // API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordIdeas const customerID = account_id.replaceAll('-', ''); const geoTargetConstants = countries[country][3]; // '2840'; const reqPayload: Record = { geoTargetConstants: `geoTargetConstants/${geoTargetConstants}`, language: `languageConstants/${language}`, pageSize: test ? '1' : '1000', }; if (['custom', 'searchconsole', 'tracking'].includes(seedType) && seedKeywords.length > 0) { reqPayload.keywordSeed = { keywords: seedKeywords.slice(0, 20) }; } if (seedType === 'auto' && domain) { reqPayload.siteSeed = { site: domain }; } const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordIdeas`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'developer-token': developer_token, Authorization: `Bearer ${accessToken}`, 'login-customer-id': customerID, }, body: JSON.stringify(reqPayload), }); const ideaData = await resp.json(); if (resp.status !== 200) { console.log('[ERROR] Google Ads Response :', ideaData?.error?.details[0]?.errors[0]?.message); // console.log('Response from Ads :', JSON.stringify(ideaData, null, 2)); } if (ideaData?.results) { fetchedKeywords = extractAdwordskeywordIdeas(ideaData.results as keywordIdeasResponseItem[], { country, domain }); } if (!test && fetchedKeywords.length > 0) { await updateLocalKeywordIdeas(domain, { keywords: fetchedKeywords, settings: adwordsDomainOptions }); } } catch (error) { console.log('[ERROR] Fetching Keyword Ideas from Google Ads :', error); } } return fetchedKeywords; }; /** * The function `extractAdwordskeywordIdeas` processes keyword ideas data and returns an array of * IdeaKeyword objects sorted by average monthly searches. * @param {keywordIdeasResponseItem[]} keywordIdeas - The `keywordIdeas` parameter is an array of * objects that contain keyword ideas and their metrics. * @param options - The `options` parameter in the `extractAdwordskeywordIdeas` function is an object * that can contain two properties: `country` and `domain`. * @returns returns an array of `IdeaKeyword` array sorted based on the average monthly searches in descending order. */ const extractAdwordskeywordIdeas = (keywordIdeas:keywordIdeasResponseItem[], options:Record) => { const keywords: IdeaKeyword[] = []; if (keywordIdeas.length > 0) { const { country = '', domain = '' } = options; keywordIdeas.forEach((kwRaw) => { const { text, keywordIdeaMetrics } = kwRaw; const { competition, competitionIndex = '0', avgMonthlySearches = '0', monthlySearchVolumes = [] } = keywordIdeaMetrics || {}; if (keywordIdeaMetrics?.avgMonthlySearches) { const searchVolumeTrend: Record = {}; const searchVolume = parseInt(avgMonthlySearches, 10); monthlySearchVolumes.forEach((item) => { searchVolumeTrend[`${item.month}-${item.year}`] = item.monthlySearches; }); if (searchVolume > 100) { keywords.push({ uid: `${country.toLowerCase()}:${text.replaceAll(' ', '-')}`, keyword: text, competition, competitionIndex: competitionIndex !== null ? parseInt(competitionIndex, 10) : 0, monthlySearchVolumes: searchVolumeTrend, avgMonthlySearches: searchVolume, added: new Date().getTime(), updated: new Date().getTime(), country, domain, position: 999, }); } } }); } return keywords.sort((a: IdeaKeyword, b: IdeaKeyword) => (b.avgMonthlySearches > a.avgMonthlySearches ? 1 : -1)); }; /** * Retrieves keyword search volumes from Google Ads API based on provided keywords and their countries. * @param {KeywordType[]} keywords - The keywords that you want to get the search volume data for. * @returns returns a Promise that resolves to an object with a `volumes` and error `proprties`. * The `volumes` propery which outputs `false` if the request fails and outputs the volume data in `{[keywordID]: volume}` object if succeeds. * The `error` porperty that outputs the error message if any. */ export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error?: string, volumes: false | Record}> => { const credentials = await getAdwordsCredentials(); if (!credentials) { return { error: 'Cannot Load Google Ads Credentials', volumes: false }; } const { client_id, client_secret, developer_token, account_id } = credentials; if (!client_id || !client_secret || !developer_token || !account_id) { return { error: 'Google Ads Not Integrated Properly', volumes: false }; } // Generate Access Token let accessToken = ''; const cachedAccessToken:string|false|undefined = memoryCache.get('adwords_token'); if (cachedAccessToken) { accessToken = cachedAccessToken; } else { accessToken = await getAdwordsAccessToken(credentials); memoryCache.delete('adwords_token'); memoryCache.set('adwords_token', accessToken, { ttl: 3300000 }); } const fetchedKeywords:Record = {}; if (accessToken) { // Group keywords based on their country. const keywordRequests: Record = {}; keywords.forEach((kw) => { const kwCountry = kw.country; if (keywordRequests[kwCountry]) { keywordRequests[kwCountry].push(kw); } else { keywordRequests[kwCountry] = [kw]; } }); // Send Requests to adwords based on grouped countries. // Since adwords does not allow sending country data for each keyword we are making requests for. for (const country in keywordRequests) { if (Object.hasOwn(keywordRequests, country) && keywordRequests[country].length > 0) { try { // API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordHistoricalMetrics const customerID = account_id.replaceAll('-', ''); const geoTargetConstants = countries[country][3]; // '2840'; const reqKeywords = keywordRequests[country].map((kw) => kw.keyword); const reqPayload: Record = { keywords: [...new Set(reqKeywords)], geoTargetConstants: `geoTargetConstants/${geoTargetConstants}`, // language: `languageConstants/${language}`, }; const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordHistoricalMetrics`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'developer-token': developer_token, Authorization: `Bearer ${accessToken}`, 'login-customer-id': customerID, }, body: JSON.stringify(reqPayload), }); const ideaData = await resp.json(); if (resp.status !== 200) { console.log('[ERROR] Google Ads Volume Request Response :', ideaData?.error?.details[0]?.errors[0]?.message); // console.log('Response from Google Ads :', JSON.stringify(ideaData, null, 2)); } if (ideaData?.results) { if (Array.isArray(ideaData.results) && ideaData.results.length > 0) { const volumeDataObj:Map = new Map(); ideaData.results.forEach((item:{ keywordMetrics: keywordIdeasMetrics, text: string }) => { const kwVol = item?.keywordMetrics?.avgMonthlySearches; volumeDataObj.set(`${country}:${item.text}`, kwVol ? parseInt(kwVol, 10) : 0); }); keywordRequests[country].forEach((keyword) => { const keywordKey = `${keyword.country}:${keyword.keyword}`; if (volumeDataObj.has(keywordKey)) { const volume = volumeDataObj.get(keywordKey); if (volume !== undefined) { fetchedKeywords[keyword.ID] = volume; } } }); // console.log('fetchedKeywords :', fetchedKeywords); } } } catch (error) { console.log('[ERROR] Fetching Keyword Volume from Google Ads :', error); } if (Object.keys(keywordRequests).length > 1) { await sleep(7000); } } } } return { volumes: fetchedKeywords }; }; /** * Updates volume data for keywords in the Keywords database using async/await and error handling. * @param {false | Record} volumesData - The `volumesData` parameter can either be `false` or an object containing * keyword IDs as keys and corresponding volume data as values. * @returns returns a Promise that resolves to `true` if `volumesData` is not `false` else it returns `false`. */ export const updateKeywordsVolumeData = async (volumesData: false | Record) => { if (volumesData === false) { return false; } Object.keys(volumesData).forEach(async (keywordID) => { const keyID = parseInt(keywordID, 10); const volumeData = volumesData && volumesData[keyID] ? volumesData[keyID] : 0; try { await Keyword.update({ volume: volumeData }, { where: { ID: keyID } }); } catch (error) { console.log(''); } }); return true; }; /** * The function `getLocalKeywordIdeas` reads keyword ideas data from a local JSON file based on a domain slug and returns it as a Promise. * @param {string} domain - The `domain` parameter is the domain slug for which the keyword Ideas are fetched. * @returns returns either a `KeywordIdeasDatabase` object if the data is successfully retrieved , or it returns `false` if * there are no keywords found in the retrieved data or if an error occurs during the process. */ export const getLocalKeywordIdeas = async (domain:string): Promise => { try { const domainName = domain.replaceAll('-', '.').replaceAll('_', '-'); const filename = `IDEAS_${domainName}.json`; const keywordIdeasRaw = await readFile(`${process.cwd()}/data/${filename}`, { encoding: 'utf-8' }); const keywordIdeasData = JSON.parse(keywordIdeasRaw) as KeywordIdeasDatabase; if (keywordIdeasData.keywords) { return keywordIdeasData; } return false; } catch (error) { // console.log('[ERROR] Getting Local Ideas. ', error); return false; } }; /** * The function `updateLocalKeywordIdeas` updates a local JSON file containing keyword ideas for a specific domain with new data provided. * @param {string} domain - The `domain` parameter is the domain slug for which the keyword Ideas are being updated. * @param {IdeaDatabaseUpdateData} data - The `data` parameter is an object of type `IdeaDatabaseUpdateData`. * It contains the following properties: `keywords`, `favorites` & `settings` * @returns The function `updateLocalKeywordIdeas` returns a Promise. */ export const updateLocalKeywordIdeas = async (domain:string, data:IdeaDatabaseUpdateData): Promise => { try { const domainName = domain.replaceAll('-', '.').replaceAll('_', '-'); const existingIdeas = await getLocalKeywordIdeas(domain); const filename = `IDEAS_${domainName}.json`; const fileContent = { ...existingIdeas, updated: new Date().getTime() }; if (data.keywords && Array.isArray(data.keywords) && data.keywords.length > 0) { fileContent.keywords = data.keywords; } if (data.favorites && Array.isArray(data.favorites) && data.favorites.length > 0) { fileContent.favorites = data.favorites; } if (data.settings) { fileContent.settings = data.settings; } await writeFile(`${process.cwd()}/data/${filename}`, JSON.stringify(fileContent, null, 2), 'utf-8'); console.log(`Data saved to ${filename} successfully!`); return true; } catch (error) { console.error(`[Error] Saving data to IDEAS_${domain}.json: ${error}`); return false; } };